{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 68869,
     "status": "ok",
     "timestamp": 1651147154902,
     "user": {
      "displayName": "YING XU",
      "userId": "11756690436182978895"
     },
     "user_tz": -480
    },
    "id": "mJjoCp1SevMW",
    "outputId": "c5a57a52-4343-4305-f45a-394c4e7acc4d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mounted at /content/drive\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "from google.colab import drive\n",
    "drive.mount('/content/drive')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "executionInfo": {
     "elapsed": 383,
     "status": "ok",
     "timestamp": 1651147159176,
     "user": {
      "displayName": "YING XU",
      "userId": "11756690436182978895"
     },
     "user_tz": -480
    },
    "id": "c_tevDwcMNac"
   },
   "outputs": [],
   "source": [
    "sys.path.append('/content/drive/MyDrive/Colab Notebooks/LexionAN-master-master')\n",
    "\n",
    "sys.argv=['']\n",
    "del sys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 9779,
     "status": "ok",
     "timestamp": 1651147169354,
     "user": {
      "displayName": "YING XU",
      "userId": "11756690436182978895"
     },
     "user_tz": -480
    },
    "id": "7AKM8H5Dfu9F",
    "outputId": "15940c26-69bb-4228-a820-5461102c46e5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting transformers\n",
      "  Downloading transformers-4.18.0-py3-none-any.whl (4.0 MB)\n",
      "\u001b[K     |████████████████████████████████| 4.0 MB 5.0 MB/s \n",
      "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
      "Collecting pyyaml>=5.1\n",
      "  Downloading PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (596 kB)\n",
      "\u001b[K     |████████████████████████████████| 596 kB 47.9 MB/s \n",
      "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.6.0)\n",
      "Collecting sacremoses\n",
      "  Downloading sacremoses-0.0.49-py3-none-any.whl (895 kB)\n",
      "\u001b[K     |████████████████████████████████| 895 kB 40.4 MB/s \n",
      "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.64.0)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
      "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.11.3)\n",
      "Collecting huggingface-hub<1.0,>=0.1.0\n",
      "  Downloading huggingface_hub-0.5.1-py3-none-any.whl (77 kB)\n",
      "\u001b[K     |████████████████████████████████| 77 kB 4.9 MB/s \n",
      "\u001b[?25hRequirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.3)\n",
      "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.21.6)\n",
      "Collecting tokenizers!=0.11.3,<0.13,>=0.11.1\n",
      "  Downloading tokenizers-0.12.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (6.6 MB)\n",
      "\u001b[K     |████████████████████████████████| 6.6 MB 34.5 MB/s \n",
      "\u001b[?25hRequirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.1.0->transformers) (4.2.0)\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers) (3.0.8)\n",
      "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.8.0)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2021.10.8)\n",
      "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
      "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.1.0)\n",
      "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
      "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
      "Installing collected packages: pyyaml, tokenizers, sacremoses, huggingface-hub, transformers\n",
      "  Attempting uninstall: pyyaml\n",
      "    Found existing installation: PyYAML 3.13\n",
      "    Uninstalling PyYAML-3.13:\n",
      "      Successfully uninstalled PyYAML-3.13\n",
      "Successfully installed huggingface-hub-0.5.1 pyyaml-6.0 sacremoses-0.0.49 tokenizers-0.12.1 transformers-4.18.0\n"
     ]
    }
   ],
   "source": [
    "!pip install transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "executionInfo": {
     "elapsed": 15669,
     "status": "ok",
     "timestamp": 1651147189773,
     "user": {
      "displayName": "YING XU",
      "userId": "11756690436182978895"
     },
     "user_tz": -480
    },
    "id": "h4OebWxdep_B"
   },
   "outputs": [],
   "source": [
    "# -*- coding: utf-8 -*-\n",
    "\n",
    "\n",
    "import time\n",
    "import sys\n",
    "import argparse\n",
    "import random\n",
    "import copy\n",
    "import torch\n",
    "import gc\n",
    "import pickle\n",
    "import json\n",
    "import os\n",
    "import torch.autograd as autograd\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "import numpy as np\n",
    "from utils.metric import get_ner_fmeasure\n",
    "from model.gazlstm import GazLSTM as SeqModel\n",
    "from utils.data import Data\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def data_initialization(data, gaz_file, train_file, dev_file, test_file):\n",
    "    data.build_alphabet(train_file)\n",
    "    data.build_alphabet(dev_file)\n",
    "    data.build_alphabet(test_file)\n",
    "    data.build_gaz_file(gaz_file)\n",
    "    data.build_gaz_alphabet(train_file,count=True)\n",
    "    data.build_gaz_alphabet(dev_file,count=True)\n",
    "    data.build_gaz_alphabet(test_file,count=True)\n",
    "    data.fix_alphabet()\n",
    "    return data\n",
    "\n",
    "\n",
    "def predict_check(pred_variable, gold_variable, mask_variable):\n",
    "    \"\"\"\n",
    "        input:\n",
    "            pred_variable (batch_size, sent_len): pred tag result, in numpy format\n",
    "            gold_variable (batch_size, sent_len): gold result variable\n",
    "            mask_variable (batch_size, sent_len): mask variable\n",
    "    \"\"\"\n",
    "\n",
    "    pred = pred_variable.cpu().data.numpy()\n",
    "    gold = gold_variable.cpu().data.numpy()\n",
    "    mask = mask_variable.cpu().data.numpy()\n",
    "    overlaped = (pred == gold)\n",
    "    right_token = np.sum(overlaped * mask)\n",
    "    total_token = mask.sum()\n",
    "\n",
    "    return right_token, total_token\n",
    "\n",
    "\n",
    "def recover_label(pred_variable, gold_variable, mask_variable, label_alphabet):\n",
    "    \"\"\"\n",
    "        input:\n",
    "            pred_variable (batch_size, sent_len): pred tag result\n",
    "            gold_variable (batch_size, sent_len): gold result variable\n",
    "            mask_variable (batch_size, sent_len): mask variable\n",
    "    \"\"\"\n",
    "    batch_size = gold_variable.size(0)\n",
    "    seq_len = gold_variable.size(1)\n",
    "    mask = mask_variable.cpu().data.numpy()\n",
    "    pred_tag = pred_variable.cpu().data.numpy()\n",
    "    gold_tag = gold_variable.cpu().data.numpy()\n",
    "    batch_size = mask.shape[0]\n",
    "    pred_label = []\n",
    "    gold_label = []\n",
    "    for idx in range(batch_size):\n",
    "        pred = [label_alphabet.get_instance(int(pred_tag[idx][idy])) for idy in range(seq_len) if mask[idx][idy] != 0]\n",
    "        gold = [label_alphabet.get_instance(gold_tag[idx][idy]) for idy in range(seq_len) if mask[idx][idy] != 0]\n",
    "\n",
    "        assert(len(pred)==len(gold))\n",
    "        pred_label.append(pred)\n",
    "        gold_label.append(gold)\n",
    "\n",
    "    return pred_label, gold_label\n",
    "\n",
    "\n",
    "def print_batchword(data, batch_word, n):\n",
    "    with open(\"labels/batchwords.txt\", \"a\") as fp:\n",
    "        for i in range(len(batch_word)):\n",
    "            words = []\n",
    "            for id in batch_word[i]:\n",
    "                words.append(data.word_alphabet.get_instance(id))\n",
    "            fp.write(str(words))\n",
    "\n",
    "def save_data_setting(data, save_file):\n",
    "    new_data = copy.deepcopy(data)\n",
    "    ## remove input instances\n",
    "    new_data.train_texts = []\n",
    "    new_data.dev_texts = []\n",
    "    new_data.test_texts = []\n",
    "    new_data.raw_texts = []\n",
    "\n",
    "    new_data.train_Ids = []\n",
    "    new_data.dev_Ids = []\n",
    "    new_data.test_Ids = []\n",
    "    new_data.raw_Ids = []\n",
    "    ## save data settings\n",
    "    with open(save_file, 'wb') as fp:\n",
    "        pickle.dump(new_data, fp)\n",
    "    print( \"Data setting saved to file: \", save_file)\n",
    "\n",
    "\n",
    "def load_data_setting(save_file):\n",
    "    with open(save_file, 'rb') as fp:\n",
    "        data = pickle.load(fp)\n",
    "    print( \"Data setting loaded from file: \", save_file)\n",
    "    data.show_data_summary()\n",
    "    return data\n",
    "\n",
    "def lr_decay(optimizer, epoch, decay_rate, init_lr):\n",
    "    lr = init_lr * ((1-decay_rate)**epoch)\n",
    "    print( \" Learning rate is setted as:\", lr)\n",
    "    for param_group in optimizer.param_groups:\n",
    "        param_group['lr'] = lr\n",
    "    return optimizer\n",
    "\n",
    "def set_seed(seed_num=1023):\n",
    "    random.seed(seed_num)\n",
    "    torch.manual_seed(seed_num)\n",
    "    np.random.seed(seed_num)\n",
    "\n",
    "\n",
    "def evaluate(data, model, name):\n",
    "    if name == \"train\":\n",
    "        instances = data.train_Ids\n",
    "    elif name == \"dev\":\n",
    "        instances = data.dev_Ids\n",
    "    elif name == 'test':\n",
    "        instances = data.test_Ids\n",
    "    elif name == 'raw':\n",
    "        instances = data.raw_Ids\n",
    "    else:\n",
    "        print( \"Error: wrong evaluate name,\", name)\n",
    "    #print(\"instances:\",instances)##\n",
    "    right_token = 0\n",
    "    whole_token = 0\n",
    "    pred_results = []\n",
    "    gold_results = []\n",
    "    ## set model in eval model\n",
    "    model.eval()\n",
    "    batch_size = 1\n",
    "    start_time = time.time()\n",
    "    train_num = len(instances)\n",
    "    total_batch = train_num//batch_size+1\n",
    "    gazes = []\n",
    "    ##print(\"train_num:\",train_num)##\n",
    "    ##print(\"total_batch:\",total_batch)##\n",
    "\n",
    "    for batch_id in range(total_batch):\n",
    "        with torch.no_grad():\n",
    "            ##print(\"hello\")##\n",
    "            start = batch_id*batch_size\n",
    "            end = (batch_id+1)*batch_size\n",
    "            if end >train_num:\n",
    "                end =  train_num\n",
    "            instance = instances[start:end]\n",
    "            if not instance:\n",
    "                continue\n",
    "            ##print(\"hello1\")##\n",
    "            gaz_list,batch_word, batch_biword, batch_wordlen, batch_label, layer_gaz, gaz_count, gaz_chars, gaz_mask, gazchar_mask, mask, batch_bert, bert_mask  = batchify_with_label(instance, data.HP_gpu, data.HP_num_layer, True)\n",
    "            ##print(\"hello2\")##\n",
    "            tag_seq, gaz_match = model(gaz_list,batch_word, batch_biword, batch_wordlen, layer_gaz, gaz_count,gaz_chars, gaz_mask, gazchar_mask, mask, batch_bert, bert_mask)\n",
    "            ##print(\"hello3\")##\n",
    "            gaz_list = [data.gaz_alphabet.get_instance(id) for batchlist in gaz_match if len(batchlist)>0 for id in batchlist ]\n",
    "            gazes.append(gaz_list)\n",
    "            ##print(\"hello4\")##\n",
    "            ##print(\"tag_seq:\",tag_seq)##\n",
    "            ##print(\"batch_label:\",batch_label)##\n",
    "            ##print(\"mask:\",mask)##\n",
    "            ##print(\"data.label_alphabet:\",data.label_alphabet)##\n",
    "\n",
    "            if name == \"dev\":\n",
    "                pred_label, gold_label = recover_label(tag_seq, batch_label, mask, data.label_alphabet)\n",
    "            else:\n",
    "                pred_label, gold_label = recover_label(tag_seq, batch_label, mask, data.label_alphabet)\n",
    "            ##print(\"pred_label:\",pred_label)##\n",
    "            ##print(\"gold_label:\",gold_label)##\n",
    "\n",
    "            pred_results += pred_label\n",
    "            gold_results += gold_label\n",
    "    decode_time = time.time() - start_time\n",
    "    if decode_time == 0:###\n",
    "        speed = 1###\n",
    "    else:###\n",
    "        speed = len(instances)/decode_time\n",
    "    # print(gold_results)\n",
    "    acc, p, r, f, right_num = get_ner_fmeasure(gold_results, pred_results, data.tagScheme)\n",
    "    return speed, acc, p, r, f, pred_results, gazes, right_num\n",
    "\n",
    "\n",
    "def get_text_input(self, caption):\n",
    "    caption_tokens = self.tokenizer.tokenize(caption)\n",
    "    caption_tokens = ['[CLS]'] + caption_tokens + ['[SEP]']\n",
    "    caption_ids = self.tokenizer.convert_tokens_to_ids(caption_tokens)\n",
    "    if len(caption_ids) >= self.max_seq_len:\n",
    "        caption_ids = caption_ids[:self.max_seq_len]\n",
    "    else:\n",
    "        caption_ids = caption_ids + [0] * (self.max_seq_len - len(caption_ids))\n",
    "    caption = torch.tensor(caption_ids)\n",
    "    return caption\n",
    "\n",
    "\n",
    "def batchify_with_label(input_batch_list, gpu, num_layer, volatile_flag=False):\n",
    "\n",
    "    batch_size = len(input_batch_list)\n",
    "    words = [sent[0] for sent in input_batch_list]\n",
    "    biwords = [sent[1] for sent in input_batch_list]\n",
    "    gazs = [sent[3] for sent in input_batch_list]\n",
    "    labels = [sent[4] for sent in input_batch_list]\n",
    "    layer_gazs = [sent[5] for sent in input_batch_list]\n",
    "    gaz_count = [sent[6] for sent in input_batch_list]\n",
    "    gaz_chars = [sent[7] for sent in input_batch_list]\n",
    "    gaz_mask = [sent[8] for sent in input_batch_list]\n",
    "    gazchar_mask = [sent[9] for sent in input_batch_list]\n",
    "    ### bert tokens\n",
    "    bert_ids = [sent[10] for sent in input_batch_list]\n",
    "\n",
    "    word_seq_lengths = torch.LongTensor(list(map(len, words)))\n",
    "    max_seq_len = word_seq_lengths.max()\n",
    "    word_seq_tensor = autograd.Variable(torch.zeros((batch_size, max_seq_len))).long()\n",
    "    biword_seq_tensor = autograd.Variable(torch.zeros((batch_size, max_seq_len))).long()\n",
    "    label_seq_tensor = autograd.Variable(torch.zeros((batch_size, max_seq_len))).long()\n",
    "    mask = autograd.Variable(torch.zeros((batch_size, max_seq_len))).byte()\n",
    "    ### bert seq tensor\n",
    "    bert_seq_tensor = autograd.Variable(torch.zeros((batch_size, max_seq_len+2))).long()\n",
    "    bert_mask = autograd.Variable(torch.zeros((batch_size, max_seq_len+2))).long()\n",
    "\n",
    "    gaz_num = [len(layer_gazs[i][0][0]) for i in range(batch_size)]\n",
    "    max_gaz_num = max(gaz_num)\n",
    "    layer_gaz_tensor = torch.zeros(batch_size, max_seq_len, 4, max_gaz_num).long()\n",
    "    gaz_count_tensor = torch.zeros(batch_size, max_seq_len, 4, max_gaz_num).float()\n",
    "    gaz_len = [len(gaz_chars[i][0][0][0]) for i in range(batch_size)]\n",
    "    max_gaz_len = max(gaz_len)\n",
    "    gaz_chars_tensor = torch.zeros(batch_size, max_seq_len, 4, max_gaz_num, max_gaz_len).long()\n",
    "    gaz_mask_tensor = torch.ones(batch_size, max_seq_len, 4, max_gaz_num).byte()\n",
    "    gazchar_mask_tensor = torch.ones(batch_size, max_seq_len, 4, max_gaz_num, max_gaz_len).byte()\n",
    "\n",
    "    for b, (seq, bert_id, biseq, label, seqlen, layergaz, gazmask, gazcount, gazchar, gazchar_mask, gaznum, gazlen) in enumerate(zip(words, bert_ids, biwords, labels, word_seq_lengths, layer_gazs, gaz_mask, gaz_count, gaz_chars, gazchar_mask, gaz_num, gaz_len)):\n",
    "\n",
    "        word_seq_tensor[b, :seqlen] = torch.LongTensor(seq)\n",
    "        biword_seq_tensor[b, :seqlen] = torch.LongTensor(biseq)\n",
    "        label_seq_tensor[b, :seqlen] = torch.LongTensor(label)\n",
    "        layer_gaz_tensor[b, :seqlen, :, :gaznum] = torch.LongTensor(layergaz)\n",
    "        mask[b, :seqlen] = torch.Tensor([1]*int(seqlen))\n",
    "        bert_mask[b, :seqlen+2] = torch.LongTensor([1]*int(seqlen+2))\n",
    "        gaz_mask_tensor[b, :seqlen, :, :gaznum] = torch.ByteTensor(gazmask)\n",
    "        gaz_count_tensor[b, :seqlen, :, :gaznum] = torch.FloatTensor(gazcount)\n",
    "        gaz_count_tensor[b, seqlen:] = 1\n",
    "        gaz_chars_tensor[b, :seqlen, :, :gaznum, :gazlen] = torch.LongTensor(gazchar)\n",
    "        gazchar_mask_tensor[b, :seqlen, :, :gaznum, :gazlen] = torch.ByteTensor(gazchar_mask)\n",
    "\n",
    "        ##bert\n",
    "        bert_seq_tensor[b, :seqlen+2] = torch.LongTensor(bert_id)\n",
    "\n",
    "\n",
    "    if gpu:\n",
    "        word_seq_tensor = word_seq_tensor.cuda()\n",
    "        biword_seq_tensor = biword_seq_tensor.cuda()\n",
    "        word_seq_lengths = word_seq_lengths.cuda()\n",
    "        label_seq_tensor = label_seq_tensor.cuda()\n",
    "        layer_gaz_tensor = layer_gaz_tensor.cuda()\n",
    "        gaz_chars_tensor = gaz_chars_tensor.cuda()\n",
    "        gaz_mask_tensor = gaz_mask_tensor.cuda()\n",
    "        gazchar_mask_tensor = gazchar_mask_tensor.cuda()\n",
    "        gaz_count_tensor = gaz_count_tensor.cuda()\n",
    "        mask = mask.cuda()\n",
    "        bert_seq_tensor = bert_seq_tensor.cuda()\n",
    "        bert_mask = bert_mask.cuda()\n",
    "\n",
    "    # print(bert_seq_tensor.type())\n",
    "    return gazs, word_seq_tensor, biword_seq_tensor, word_seq_lengths, label_seq_tensor, layer_gaz_tensor, gaz_count_tensor,gaz_chars_tensor, gaz_mask_tensor, gazchar_mask_tensor, mask, bert_seq_tensor, bert_mask\n",
    "\n",
    "\n",
    "\n",
    "def train(data, save_model_dir, seg=True):\n",
    "    train_log = []\n",
    "    dev_log = []\n",
    "    test_log = []\n",
    "\n",
    "    print(\"Training with {} model.\".format(data.model_type))\n",
    "\n",
    "    #data.show_data_summary()\n",
    "\n",
    "\n",
    "    model = SeqModel(data)\n",
    "    print( \"finish building model.\")\n",
    "\n",
    "    parameters = filter(lambda p: p.requires_grad, model.parameters())\n",
    "    optimizer = optim.Adamax(parameters, lr=data.HP_lr)\n",
    "\n",
    "    best_dev = -1\n",
    "    best_dev_p = -1\n",
    "    best_dev_r = -1\n",
    "\n",
    "    best_test = -1\n",
    "    best_test_p = -1\n",
    "    best_test_r = -1\n",
    "\n",
    "\n",
    "    ## start training\n",
    "    for idx in range(data.HP_iteration):\n",
    "        epoch_start = time.time()\n",
    "        temp_start = epoch_start\n",
    "        print((\"Epoch: %s/%s\" %(idx,data.HP_iteration)))\n",
    "        optimizer = lr_decay(optimizer, idx, data.HP_lr_decay, data.HP_lr)\n",
    "        instance_count = 0\n",
    "        sample_loss = 0\n",
    "        batch_loss = 0\n",
    "        total_loss = 0\n",
    "        right_token = 0\n",
    "        whole_token = 0\n",
    "        random.shuffle(data.train_Ids)\n",
    "        ## set model in train model\n",
    "        model.train()\n",
    "        model.zero_grad()\n",
    "        batch_size = data.HP_batch_size\n",
    "        batch_id = 0\n",
    "        train_num = len(data.train_Ids)\n",
    "        total_batch = train_num//batch_size+1\n",
    "\n",
    "        for batch_id in range(total_batch):\n",
    "            start = batch_id*batch_size\n",
    "            end = (batch_id+1)*batch_size\n",
    "            if end >train_num:\n",
    "                end = train_num\n",
    "            instance = data.train_Ids[start:end]\n",
    "            words = data.train_texts[start:end]\n",
    "            if not instance:\n",
    "                continue\n",
    "\n",
    "            gaz_list,  batch_word, batch_biword, batch_wordlen, batch_label, layer_gaz, gaz_count, gaz_chars, gaz_mask, gazchar_mask, mask, batch_bert, bert_mask = batchify_with_label(instance, data.HP_gpu,data.HP_num_layer)\n",
    "\n",
    "            instance_count += 1\n",
    "            loss, tag_seq = model.neg_log_likelihood_loss(gaz_list, batch_word, batch_biword, batch_wordlen, layer_gaz, gaz_count,gaz_chars, gaz_mask, gazchar_mask, mask, batch_label, batch_bert, bert_mask)\n",
    "\n",
    "            right, whole = predict_check(tag_seq, batch_label, mask)\n",
    "            right_token += right\n",
    "            whole_token += whole\n",
    "            sample_loss += loss.data\n",
    "            total_loss += loss.data\n",
    "            batch_loss += loss\n",
    "\n",
    "            if end%20 == 0:\n",
    "                temp_time = time.time()\n",
    "                temp_cost = temp_time - temp_start\n",
    "                temp_start = temp_time\n",
    "                print((\"Instance: %s; Time: %.2fs; loss: %.4f; acc: %s/%s=%.4f\"%(end, temp_cost, sample_loss, right_token, whole_token,(right_token+0.)/whole_token)))\n",
    "                sys.stdout.flush()\n",
    "                sample_loss = 0\n",
    "            if end%data.HP_batch_size == 0:\n",
    "                batch_loss.backward()\n",
    "                optimizer.step()\n",
    "                model.zero_grad()\n",
    "                batch_loss = 0\n",
    "        train_log.append(total_loss.item())\n",
    "\n",
    "        temp_time = time.time()\n",
    "        temp_cost = temp_time - temp_start\n",
    "        print((\"     Instance: %s; Time: %.2fs; loss: %.4f; acc: %s/%s=%.4f\"%(end, temp_cost, sample_loss, right_token, whole_token,(right_token+0.)/whole_token))       )\n",
    "        epoch_finish = time.time()\n",
    "        epoch_cost = epoch_finish - epoch_start\n",
    "        print((\"Epoch: %s training finished. Time: %.2fs, speed: %.2fst/s,  total loss: %s\"%(idx, epoch_cost, train_num/epoch_cost, total_loss)))\n",
    "\n",
    "        speed, acc, p, r, f, pred_labels, gazs,right_num = evaluate(data, model, \"dev\")\n",
    "        \n",
    "        dev_instance = data.dev_Ids\n",
    "        dev_gaz_list, dev_batch_word, dev_batch_biword, dev_batch_wordlen, dev_batch_label, dev_layer_gaz, dev_gaz_count, dev_gaz_chars, dev_gaz_mask, dev_gazchar_mask, dev_mask, dev_batch_bert, dev_bert_mask = batchify_with_label(dev_instance, data.HP_gpu,data.HP_num_layer)\n",
    "        dev_loss, dev_tag_seq = model.neg_log_likelihood_loss(dev_gaz_list, dev_batch_word, dev_batch_biword, dev_batch_wordlen, dev_layer_gaz, dev_gaz_count,dev_gaz_chars, dev_gaz_mask, dev_gazchar_mask, dev_mask, dev_batch_label, dev_batch_bert, dev_bert_mask)\n",
    "        print('这是dev_loss：',dev_loss)\n",
    "        \n",
    "        dev_log.append(dev_loss.item())\n",
    "        dev_finish = time.time()\n",
    "        dev_cost = dev_finish - epoch_finish\n",
    "\n",
    "        if seg:\n",
    "            current_score = f\n",
    "            print((\"Dev: time: %.2fs, speed: %.2fst/s; acc: %.4f, p: %.4f, r: %.4f, f: %.4f\"%(dev_cost, speed, acc, p, r, f)))\n",
    "        else:\n",
    "            current_score = acc\n",
    "            print((\"Dev: time: %.2fs speed: %.2fst/s; acc: %.4f\"%(dev_cost, speed, acc)))\n",
    "\n",
    "        if current_score > best_dev:\n",
    "            if seg:\n",
    "                print( \"Exceed previous best f score:\", best_dev)\n",
    "\n",
    "            else:\n",
    "                print( \"Exceed previous best acc score:\", best_dev)\n",
    "\n",
    "            model_name = save_model_dir\n",
    "            torch.save(model.state_dict(), model_name)\n",
    "            #best_dev = current_score\n",
    "            best_dev_p = p\n",
    "            best_dev_r = r\n",
    "\n",
    "        # ## decode test\n",
    "        speed, acc, p, r, f, pred_labels, gazs,right_num = evaluate(data, model, \"test\")\n",
    "        test_log.append(pred_labels)\n",
    "        test_finish = time.time()\n",
    "        test_cost = test_finish - dev_finish\n",
    "        if seg:\n",
    "            current_test_score = f\n",
    "            print((\"Test: time: %.2fs, speed: %.2fst/s; acc: %.4f, p: %.4f, r: %.4f, f: %.4f\"%(test_cost, speed, acc, p, r, f)))\n",
    "        else:\n",
    "            current_test_score = acc\n",
    "            print((\"Test: time: %.2fs, speed: %.2fst/s; acc: %.4f\"%(test_cost, speed, acc)))\n",
    "\n",
    "        if current_score > best_dev:\n",
    "            best_dev = current_score\n",
    "            best_test = current_test_score\n",
    "            best_test_p = p\n",
    "            best_test_r = r\n",
    "\n",
    "        print(\"Best dev score: p:{}, r:{}, f:{}\".format(best_dev_p,best_dev_r,best_dev))\n",
    "        print(\"Test score: p:{}, r:{}, f:{}\".format(best_test_p,best_test_r,best_test))\n",
    "        gc.collect()\n",
    "\n",
    "    with open(data.result_file,\"a\") as f:\n",
    "        f.write(save_model_dir+'\\n')\n",
    "        f.write(\"Best dev score: p:{}, r:{}, f:{}\\n\".format(best_dev_p,best_dev_r,best_dev))\n",
    "        f.write(\"Test score: p:{}, r:{}, f:{}\\n\\n\".format(best_test_p,best_test_r,best_test))\n",
    "        f.close()\n",
    "\n",
    "    return model,train_log,dev_log,test_log\n",
    "\n",
    "def load_model_decode(model_dir, data, name, gpu, seg=True):\n",
    "    data.HP_gpu = gpu\n",
    "    print( \"Load Model from file: \", model_dir)\n",
    "    model = SeqModel(data)\n",
    "\n",
    "    model.load_state_dict(torch.load(model_dir))\n",
    "\n",
    "    print((\"Decode %s data ...\"%(name)))\n",
    "    start_time = time.time()\n",
    "    speed, acc, p, r, f, pred_results, gazs = evaluate(data, model, name)\n",
    "    end_time = time.time()\n",
    "    time_cost = end_time - start_time\n",
    "    if seg:\n",
    "        print((\"%s: time:%.2fs, speed:%.2fst/s; acc: %.4f, p: %.4f, r: %.4f, f: %.4f\"%(name, time_cost, speed, acc, p, r, f)))\n",
    "    else:\n",
    "        print((\"%s: time:%.2fs, speed:%.2fst/s; acc: %.4f\"%(name, time_cost, speed, acc)))\n",
    "\n",
    "    return pred_results\n",
    "\n",
    "\n",
    "def print_results(pred, modelname=\"\"):\n",
    "    toprint = []\n",
    "    for sen in pred:\n",
    "        sen = \" \".join(sen) + '\\n'\n",
    "        toprint.append(sen)\n",
    "    with open(modelname+'_labels.txt','w') as f:\n",
    "        f.writelines(toprint)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 130679,
     "status": "ok",
     "timestamp": 1651149529406,
     "user": {
      "displayName": "YING XU",
      "userId": "11756690436182978895"
     },
     "user_tz": -480
    },
    "id": "64fL85GJfpm-",
    "outputId": "871631c2-1800-49c2-b9bb-905d8605fec9"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hello world\n",
      "Load gaz file:  /content/drive/MyDrive/Colab Notebooks/LexionAN-master-master/data/TCM_w2v.vec  total size: 63896\n",
      "gaz alphabet size: 2055\n",
      "gaz alphabet size: 2179\n",
      "gaz alphabet size: 2376\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub data rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_data_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_data_rate_limit=1000000.0 (bytes/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "build word pretrain emb...\n",
      "Embedding:\n",
      "     pretrain word:19527, prefect match:965, case_match:1, oov:12, oov%:0.012257405515832482\n",
      "build biword pretrain emb...\n",
      "Embedding:\n",
      "     pretrain word:19527, prefect match:0, case_match:0, oov:5529, oov%:0.9998191681735985\n",
      "build gaz pretrain emb...\n",
      "Embedding:\n",
      "     pretrain word:63896, prefect match:2374, case_match:0, oov:1, oov%:0.00042087542087542086\n",
      "Dumping data\n",
      "data.use_biword= False\n",
      "Training with lstm model.\n",
      "build batched crf...\n",
      "finish building model.\n",
      "Epoch: 0/16\n",
      " Learning rate is setted as: 0.005\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/content/drive/MyDrive/Colab Notebooks/LexionAN-master-master/model/gazlstm.py:146: UserWarning: masked_fill_ received a mask with dtype torch.uint8, this behavior is now deprecated,please use a mask with dtype torch.bool instead. (Triggered internally at  ../aten/src/ATen/native/cuda/Indexing.cu:967.)\n",
      "  gaz_embeds = gaz_embeds_d.data.masked_fill_(gaz_mask.data, 0)  #(b,l,4,g,ge)  ge:gaz_embed_dim\n",
      "/content/drive/MyDrive/Colab Notebooks/LexionAN-master-master/model/crf.py:95: UserWarning: indexing with dtype torch.uint8 is now deprecated, please use a dtype torch.bool instead. (Triggered internally at  ../aten/src/ATen/native/IndexingUtils.h:28.)\n",
      "  masked_cur_partition = cur_partition.masked_select(mask_idx)\n",
      "/content/drive/MyDrive/Colab Notebooks/LexionAN-master-master/model/crf.py:100: UserWarning: masked_scatter_ received a mask with dtype torch.uint8, this behavior is now deprecated,please use a mask with dtype torch.bool instead. (Triggered internally at  ../aten/src/ATen/native/cuda/IndexKernel.cpp:62.)\n",
      "  partition.masked_scatter_(mask_idx, masked_cur_partition)\n",
      "/content/drive/MyDrive/Colab Notebooks/LexionAN-master-master/model/crf.py:245: UserWarning: indexing with dtype torch.uint8 is now deprecated, please use a dtype torch.bool instead. (Triggered internally at  ../aten/src/ATen/native/IndexingUtils.h:28.)\n",
      "  tg_energy = tg_energy.masked_select(mask.transpose(1,0))\n",
      "/content/drive/MyDrive/Colab Notebooks/LexionAN-master-master/model/crf.py:159: UserWarning: masked_fill_ received a mask with dtype torch.uint8, this behavior is now deprecated,please use a mask with dtype torch.bool instead. (Triggered internally at  ../aten/src/ATen/native/cuda/Indexing.cu:967.)\n",
      "  cur_bp.masked_fill_(mask[idx].view(batch_size, 1).expand(batch_size, tag_size), 0)\n",
      "/usr/local/lib/python3.7/dist-packages/torch/autograd/__init__.py:175: UserWarning: masked_scatter_ received a mask with dtype torch.uint8, this behavior is now deprecated,please use a mask with dtype torch.bool instead. (Triggered internally at  ../aten/src/ATen/native/cuda/IndexKernel.cpp:62.)\n",
      "  allow_unreachable=True, accumulate_grad=True)  # Calls into the C++ engine to run the backward pass\n",
      "/usr/local/lib/python3.7/dist-packages/torch/autograd/__init__.py:175: UserWarning: masked_fill_ received a mask with dtype torch.uint8, this behavior is now deprecated,please use a mask with dtype torch.bool instead. (Triggered internally at  ../aten/src/ATen/native/cuda/Indexing.cu:967.)\n",
      "  allow_unreachable=True, accumulate_grad=True)  # Calls into the C++ engine to run the backward pass\n",
      "/usr/local/lib/python3.7/dist-packages/torch/autograd/__init__.py:175: UserWarning: indexing with dtype torch.uint8 is now deprecated, please use a dtype torch.bool instead. (Triggered internally at  ../aten/src/ATen/native/IndexingUtils.h:28.)\n",
      "  allow_unreachable=True, accumulate_grad=True)  # Calls into the C++ engine to run the backward pass\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Instance: 20; Time: 0.22s; loss: 1158.7458; acc: 56.0/400.0=0.1400\n",
      "Instance: 40; Time: 0.15s; loss: 954.9408; acc: 191.0/768.0=0.2487\n",
      "Instance: 60; Time: 0.16s; loss: 1043.3889; acc: 341.0/1212.0=0.2814\n",
      "Instance: 80; Time: 0.23s; loss: 1032.7479; acc: 547.0/1717.0=0.3186\n",
      "Instance: 100; Time: 0.19s; loss: 748.6099; acc: 683.0/2094.0=0.3262\n",
      "Instance: 120; Time: 0.24s; loss: 906.8417; acc: 880.0/2567.0=0.3428\n",
      "Instance: 140; Time: 0.23s; loss: 665.8825; acc: 1063.0/2955.0=0.3597\n",
      "Instance: 160; Time: 0.17s; loss: 804.6241; acc: 1304.0/3432.0=0.3800\n",
      "Instance: 180; Time: 0.26s; loss: 988.6688; acc: 1580.0/4003.0=0.3947\n",
      "Instance: 200; Time: 0.22s; loss: 666.5951; acc: 1784.0/4424.0=0.4033\n",
      "Instance: 220; Time: 0.16s; loss: 528.8035; acc: 1972.0/4769.0=0.4135\n",
      "Instance: 240; Time: 0.17s; loss: 626.3945; acc: 2214.0/5190.0=0.4266\n",
      "Instance: 260; Time: 0.21s; loss: 709.9688; acc: 2518.0/5732.0=0.4393\n",
      "Instance: 280; Time: 0.18s; loss: 515.0520; acc: 2816.0/6179.0=0.4557\n",
      "Instance: 300; Time: 0.17s; loss: 559.2933; acc: 3127.0/6658.0=0.4697\n",
      "Instance: 320; Time: 0.15s; loss: 557.9229; acc: 3398.0/7107.0=0.4781\n",
      "Instance: 340; Time: 0.26s; loss: 553.4088; acc: 3712.0/7578.0=0.4898\n",
      "Instance: 360; Time: 0.25s; loss: 478.9127; acc: 4027.0/8046.0=0.5005\n",
      "Instance: 380; Time: 0.20s; loss: 553.5703; acc: 4363.0/8543.0=0.5107\n",
      "Instance: 400; Time: 0.25s; loss: 529.1257; acc: 4767.0/9094.0=0.5242\n",
      "Instance: 420; Time: 0.17s; loss: 402.4144; acc: 4984.0/9430.0=0.5285\n",
      "Instance: 440; Time: 0.15s; loss: 302.9324; acc: 5242.0/9777.0=0.5362\n",
      "Instance: 460; Time: 0.18s; loss: 525.9321; acc: 5544.0/10245.0=0.5411\n",
      "Instance: 480; Time: 0.15s; loss: 393.0819; acc: 5832.0/10648.0=0.5477\n",
      "Instance: 500; Time: 0.16s; loss: 380.4434; acc: 6114.0/11049.0=0.5534\n",
      "     Instance: 501; Time: 0.08s; loss: 37.5176; acc: 6130.0/11074.0=0.5535\n",
      "Epoch: 0 training finished. Time: 4.95s, speed: 101.12st/s,  total loss: tensor(16625.8184, device='cuda:0')\n",
      "gold_num =  341  pred_num =  291  right_num =  161\n",
      "这是dev_loss： tensor(1244.9326, device='cuda:0', grad_fn=<SubBackward0>)\n",
      "Dev: time: 0.69s, speed: 131.12st/s; acc: 0.7560, p: 0.5533, r: 0.4721, f: 0.5095\n",
      "Exceed previous best f score: -1\n",
      "gold_num =  697  pred_num =  603  right_num =  348\n",
      "Test: time: 1.11s, speed: 112.38st/s; acc: 0.7479, p: 0.5771, r: 0.4993, f: 0.5354\n",
      "Best dev score: p:0.5532646048109966, r:0.47214076246334313, f:0.5094936708860759\n",
      "Test score: p:0.5771144278606966, r:0.49928263988522237, f:0.5353846153846153\n",
      "Epoch: 1/16\n",
      " Learning rate is setted as: 0.00475\n",
      "Instance: 20; Time: 0.13s; loss: 301.5465; acc: 263.0/361.0=0.7285\n",
      "Instance: 40; Time: 0.15s; loss: 337.0621; acc: 467.0/682.0=0.6848\n",
      "Instance: 60; Time: 0.15s; loss: 393.4828; acc: 736.0/1063.0=0.6924\n",
      "Instance: 80; Time: 0.22s; loss: 438.3246; acc: 1142.0/1596.0=0.7155\n",
      "Instance: 100; Time: 0.18s; loss: 376.8195; acc: 1470.0/2024.0=0.7263\n",
      "Instance: 120; Time: 0.27s; loss: 301.0143; acc: 1789.0/2434.0=0.7350\n",
      "Instance: 140; Time: 0.29s; loss: 392.3569; acc: 2191.0/2941.0=0.7450\n",
      "Instance: 160; Time: 0.17s; loss: 232.3087; acc: 2534.0/3337.0=0.7594\n",
      "Instance: 180; Time: 0.17s; loss: 343.3439; acc: 2867.0/3774.0=0.7597\n",
      "Instance: 200; Time: 0.20s; loss: 371.6758; acc: 3202.0/4216.0=0.7595\n",
      "Instance: 220; Time: 0.26s; loss: 361.4171; acc: 3540.0/4655.0=0.7605\n",
      "Instance: 240; Time: 0.30s; loss: 256.7051; acc: 3873.0/5068.0=0.7642\n",
      "Instance: 260; Time: 0.18s; loss: 319.7108; acc: 4250.0/5537.0=0.7676\n",
      "Instance: 280; Time: 0.18s; loss: 381.0396; acc: 4568.0/5975.0=0.7645\n",
      "Instance: 300; Time: 0.24s; loss: 330.4937; acc: 5044.0/6535.0=0.7718\n",
      "Instance: 320; Time: 0.31s; loss: 399.4086; acc: 5551.0/7153.0=0.7760\n",
      "Instance: 340; Time: 0.20s; loss: 226.2515; acc: 5859.0/7532.0=0.7779\n",
      "Instance: 360; Time: 0.18s; loss: 283.5231; acc: 6192.0/7952.0=0.7787\n",
      "Instance: 380; Time: 0.17s; loss: 221.8491; acc: 6583.0/8397.0=0.7840\n",
      "Instance: 400; Time: 0.17s; loss: 220.6583; acc: 6915.0/8785.0=0.7871\n",
      "Instance: 420; Time: 0.23s; loss: 296.6879; acc: 7286.0/9228.0=0.7896\n",
      "Instance: 440; Time: 0.27s; loss: 246.1414; acc: 7614.0/9623.0=0.7912\n",
      "Instance: 460; Time: 0.20s; loss: 243.2374; acc: 7974.0/10053.0=0.7932\n",
      "Instance: 480; Time: 0.17s; loss: 233.6842; acc: 8403.0/10540.0=0.7972\n",
      "Instance: 500; Time: 0.21s; loss: 298.6068; acc: 8837.0/11064.0=0.7987\n",
      "     Instance: 501; Time: 0.06s; loss: 3.4754; acc: 8846.0/11074.0=0.7988\n",
      "Epoch: 1 training finished. Time: 5.25s, speed: 95.50st/s,  total loss: tensor(7810.8257, device='cuda:0')\n",
      "gold_num =  341  pred_num =  348  right_num =  226\n",
      "这是dev_loss： tensor(900.6777, device='cuda:0', grad_fn=<SubBackward0>)\n",
      "Dev: time: 0.67s, speed: 135.68st/s; acc: 0.8327, p: 0.6494, r: 0.6628, f: 0.6560\n",
      "Exceed previous best f score: 0.5094936708860759\n",
      "gold_num =  697  pred_num =  711  right_num =  484\n",
      "Test: time: 1.11s, speed: 112.08st/s; acc: 0.8288, p: 0.6807, r: 0.6944, f: 0.6875\n",
      "Best dev score: p:0.6494252873563219, r:0.6627565982404692, f:0.6560232220609579\n",
      "Test score: p:0.680731364275668, r:0.6944045911047346, f:0.6875000000000001\n",
      "Epoch: 2/16\n",
      " Learning rate is setted as: 0.0045125\n",
      "Instance: 20; Time: 0.14s; loss: 193.1910; acc: 323.0/377.0=0.8568\n",
      "Instance: 40; Time: 0.21s; loss: 305.2173; acc: 729.0/874.0=0.8341\n",
      "Instance: 60; Time: 0.33s; loss: 232.9243; acc: 1101.0/1314.0=0.8379\n",
      "Instance: 80; Time: 0.18s; loss: 201.5318; acc: 1419.0/1683.0=0.8431\n",
      "Instance: 100; Time: 0.15s; loss: 260.3932; acc: 1799.0/2143.0=0.8395\n",
      "Instance: 120; Time: 0.19s; loss: 219.4189; acc: 2110.0/2524.0=0.8360\n",
      "Instance: 140; Time: 0.21s; loss: 224.5074; acc: 2517.0/2997.0=0.8398\n",
      "Instance: 160; Time: 0.17s; loss: 187.6518; acc: 2825.0/3357.0=0.8415\n",
      "Instance: 180; Time: 0.16s; loss: 213.6210; acc: 3152.0/3744.0=0.8419\n",
      "Instance: 200; Time: 0.15s; loss: 184.6282; acc: 3499.0/4137.0=0.8458\n",
      "Instance: 220; Time: 0.16s; loss: 181.5846; acc: 3833.0/4525.0=0.8471\n",
      "Instance: 240; Time: 0.18s; loss: 233.8495; acc: 4208.0/4966.0=0.8474\n",
      "Instance: 260; Time: 0.18s; loss: 189.7324; acc: 4531.0/5337.0=0.8490\n",
      "Instance: 280; Time: 0.25s; loss: 219.8121; acc: 5009.0/5876.0=0.8525\n",
      "Instance: 300; Time: 0.19s; loss: 153.8308; acc: 5414.0/6322.0=0.8564\n",
      "Instance: 320; Time: 0.18s; loss: 274.5422; acc: 5847.0/6833.0=0.8557\n",
      "Instance: 340; Time: 0.21s; loss: 168.7382; acc: 6244.0/7282.0=0.8575\n",
      "Instance: 360; Time: 0.27s; loss: 323.1424; acc: 6663.0/7785.0=0.8559\n",
      "Instance: 380; Time: 0.24s; loss: 204.3574; acc: 7090.0/8276.0=0.8567\n",
      "Instance: 400; Time: 0.15s; loss: 209.4395; acc: 7503.0/8750.0=0.8575\n",
      "Instance: 420; Time: 0.18s; loss: 278.5592; acc: 7923.0/9245.0=0.8570\n",
      "Instance: 440; Time: 0.22s; loss: 332.2621; acc: 8328.0/9734.0=0.8556\n",
      "Instance: 460; Time: 0.21s; loss: 210.1687; acc: 8769.0/10229.0=0.8573\n",
      "Instance: 480; Time: 0.25s; loss: 253.0092; acc: 9183.0/10711.0=0.8573\n",
      "Instance: 500; Time: 0.18s; loss: 119.1948; acc: 9506.0/11064.0=0.8592\n",
      "     Instance: 501; Time: 0.05s; loss: 3.1233; acc: 9516.0/11074.0=0.8593\n",
      "Epoch: 2 training finished. Time: 4.99s, speed: 100.33st/s,  total loss: tensor(5578.4302, device='cuda:0')\n",
      "gold_num =  341  pred_num =  353  right_num =  253\n",
      "这是dev_loss： tensor(773.9131, device='cuda:0', grad_fn=<SubBackward0>)\n",
      "Dev: time: 0.67s, speed: 135.21st/s; acc: 0.8511, p: 0.7167, r: 0.7419, f: 0.7291\n",
      "Exceed previous best f score: 0.6560232220609579\n",
      "gold_num =  697  pred_num =  733  right_num =  524\n",
      "Test: time: 1.13s, speed: 109.49st/s; acc: 0.8489, p: 0.7149, r: 0.7518, f: 0.7329\n",
      "Best dev score: p:0.71671388101983, r:0.7419354838709677, f:0.7291066282420748\n",
      "Test score: p:0.7148703956343793, r:0.7517934002869441, f:0.7328671328671329\n",
      "Epoch: 3/16\n",
      " Learning rate is setted as: 0.004286875\n",
      "Instance: 20; Time: 0.13s; loss: 202.5675; acc: 395.0/450.0=0.8778\n",
      "Instance: 40; Time: 0.16s; loss: 169.3267; acc: 761.0/859.0=0.8859\n",
      "Instance: 60; Time: 0.14s; loss: 246.1243; acc: 1086.0/1261.0=0.8612\n",
      "Instance: 80; Time: 0.19s; loss: 215.1469; acc: 1566.0/1799.0=0.8705\n",
      "Instance: 100; Time: 0.30s; loss: 251.9705; acc: 2089.0/2394.0=0.8726\n",
      "Instance: 120; Time: 0.19s; loss: 141.8942; acc: 2481.0/2823.0=0.8789\n",
      "Instance: 140; Time: 0.21s; loss: 160.2750; acc: 2940.0/3320.0=0.8855\n",
      "Instance: 160; Time: 0.15s; loss: 89.7053; acc: 3226.0/3626.0=0.8897\n",
      "Instance: 180; Time: 0.17s; loss: 196.8287; acc: 3609.0/4067.0=0.8874\n",
      "Instance: 200; Time: 0.24s; loss: 252.6836; acc: 4109.0/4644.0=0.8848\n",
      "Instance: 220; Time: 0.18s; loss: 107.0494; acc: 4487.0/5046.0=0.8892\n",
      "Instance: 240; Time: 0.18s; loss: 165.9706; acc: 4839.0/5445.0=0.8887\n",
      "Instance: 260; Time: 0.22s; loss: 221.6278; acc: 5235.0/5903.0=0.8868\n",
      "Instance: 280; Time: 0.19s; loss: 118.8887; acc: 5579.0/6275.0=0.8891\n",
      "Instance: 300; Time: 0.17s; loss: 145.4332; acc: 5937.0/6665.0=0.8908\n",
      "Instance: 320; Time: 0.13s; loss: 169.1519; acc: 6193.0/6980.0=0.8872\n",
      "Instance: 340; Time: 0.23s; loss: 151.5118; acc: 6716.0/7544.0=0.8902\n",
      "Instance: 360; Time: 0.21s; loss: 212.9816; acc: 7098.0/7995.0=0.8878\n",
      "Instance: 380; Time: 0.19s; loss: 227.5979; acc: 7443.0/8404.0=0.8856\n",
      "Instance: 400; Time: 0.13s; loss: 106.7064; acc: 7806.0/8796.0=0.8874\n",
      "Instance: 420; Time: 0.17s; loss: 198.4827; acc: 8190.0/9239.0=0.8865\n",
      "Instance: 440; Time: 0.16s; loss: 111.3004; acc: 8565.0/9646.0=0.8879\n",
      "Instance: 460; Time: 0.18s; loss: 201.7350; acc: 8990.0/10118.0=0.8885\n",
      "Instance: 480; Time: 0.25s; loss: 195.4313; acc: 9408.0/10588.0=0.8886\n",
      "Instance: 500; Time: 0.25s; loss: 147.7202; acc: 9833.0/11053.0=0.8896\n",
      "     Instance: 501; Time: 0.11s; loss: 9.1760; acc: 9851.0/11074.0=0.8896\n",
      "Epoch: 3 training finished. Time: 4.80s, speed: 104.44st/s,  total loss: tensor(4417.2876, device='cuda:0')\n",
      "gold_num =  341  pred_num =  353  right_num =  265\n",
      "这是dev_loss： tensor(681.8301, device='cuda:0', grad_fn=<SubBackward0>)\n",
      "Dev: time: 0.71s, speed: 130.38st/s; acc: 0.8726, p: 0.7507, r: 0.7771, f: 0.7637\n",
      "Exceed previous best f score: 0.7291066282420748\n",
      "gold_num =  697  pred_num =  718  right_num =  538\n",
      "Test: time: 1.14s, speed: 108.73st/s; acc: 0.8593, p: 0.7493, r: 0.7719, f: 0.7604\n",
      "Best dev score: p:0.7507082152974505, r:0.7771260997067448, f:0.7636887608069165\n",
      "Test score: p:0.7493036211699164, r:0.7718794835007173, f:0.7604240282685514\n",
      "Epoch: 4/16\n",
      " Learning rate is setted as: 0.00407253125\n",
      "Instance: 20; Time: 0.14s; loss: 139.3495; acc: 394.0/432.0=0.9120\n",
      "Instance: 40; Time: 0.22s; loss: 201.9683; acc: 871.0/971.0=0.8970\n",
      "Instance: 60; Time: 0.19s; loss: 100.9053; acc: 1229.0/1354.0=0.9077\n",
      "Instance: 80; Time: 0.23s; loss: 150.9709; acc: 1649.0/1816.0=0.9080\n",
      "Instance: 100; Time: 0.16s; loss: 125.9178; acc: 2002.0/2204.0=0.9083\n",
      "Instance: 120; Time: 0.19s; loss: 140.3574; acc: 2390.0/2629.0=0.9091\n",
      "Instance: 140; Time: 0.18s; loss: 161.8298; acc: 2832.0/3114.0=0.9094\n",
      "Instance: 160; Time: 0.15s; loss: 138.1605; acc: 3244.0/3563.0=0.9105\n",
      "Instance: 180; Time: 0.19s; loss: 124.9171; acc: 3618.0/3969.0=0.9116\n",
      "Instance: 200; Time: 0.27s; loss: 225.5341; acc: 4220.0/4633.0=0.9109\n",
      "Instance: 220; Time: 0.21s; loss: 119.9317; acc: 4545.0/4994.0=0.9101\n",
      "Instance: 240; Time: 0.18s; loss: 163.9926; acc: 4847.0/5344.0=0.9070\n",
      "Instance: 260; Time: 0.19s; loss: 103.3988; acc: 5227.0/5752.0=0.9087\n",
      "Instance: 280; Time: 0.23s; loss: 130.3614; acc: 5592.0/6157.0=0.9082\n",
      "Instance: 300; Time: 0.15s; loss: 156.2924; acc: 5862.0/6477.0=0.9050\n",
      "Instance: 320; Time: 0.20s; loss: 133.7715; acc: 6285.0/6934.0=0.9064\n",
      "Instance: 340; Time: 0.24s; loss: 164.5573; acc: 6749.0/7439.0=0.9072\n",
      "Instance: 360; Time: 0.27s; loss: 292.7205; acc: 7187.0/7955.0=0.9035\n",
      "Instance: 380; Time: 0.19s; loss: 126.9758; acc: 7576.0/8382.0=0.9038\n",
      "Instance: 400; Time: 0.17s; loss: 126.3795; acc: 7970.0/8815.0=0.9041\n",
      "Instance: 420; Time: 0.13s; loss: 164.2727; acc: 8319.0/9217.0=0.9026\n",
      "Instance: 440; Time: 0.21s; loss: 103.4493; acc: 8713.0/9636.0=0.9042\n",
      "Instance: 460; Time: 0.23s; loss: 188.8616; acc: 9188.0/10163.0=0.9041\n",
      "Instance: 480; Time: 0.19s; loss: 130.1542; acc: 9524.0/10540.0=0.9036\n",
      "Instance: 500; Time: 0.30s; loss: 161.3687; acc: 9959.0/11016.0=0.9040\n",
      "     Instance: 501; Time: 0.10s; loss: 19.2007; acc: 10013.0/11074.0=0.9042\n",
      "Epoch: 4 training finished. Time: 5.10s, speed: 98.25st/s,  total loss: tensor(3795.5996, device='cuda:0')\n",
      "gold_num =  341  pred_num =  354  right_num =  263\n",
      "这是dev_loss： tensor(648.6494, device='cuda:0', grad_fn=<SubBackward0>)\n",
      "Dev: time: 0.69s, speed: 132.18st/s; acc: 0.8733, p: 0.7429, r: 0.7713, f: 0.7568\n",
      "gold_num =  697  pred_num =  707  right_num =  545\n",
      "Test: time: 1.10s, speed: 109.47st/s; acc: 0.8656, p: 0.7709, r: 0.7819, f: 0.7764\n",
      "Best dev score: p:0.7507082152974505, r:0.7771260997067448, f:0.7636887608069165\n",
      "Test score: p:0.7493036211699164, r:0.7718794835007173, f:0.7604240282685514\n",
      "Epoch: 5/16\n",
      " Learning rate is setted as: 0.003868904687499999\n",
      "Instance: 20; Time: 0.13s; loss: 173.7385; acc: 345.0/393.0=0.8779\n",
      "Instance: 40; Time: 0.15s; loss: 89.9946; acc: 736.0/809.0=0.9098\n",
      "Instance: 60; Time: 0.22s; loss: 134.1060; acc: 1199.0/1314.0=0.9125\n",
      "Instance: 80; Time: 0.22s; loss: 149.4435; acc: 1635.0/1786.0=0.9155\n",
      "Instance: 100; Time: 0.17s; loss: 110.6835; acc: 1990.0/2170.0=0.9171\n",
      "Instance: 120; Time: 0.24s; loss: 186.5697; acc: 2521.0/2753.0=0.9157\n",
      "Instance: 140; Time: 0.19s; loss: 99.2216; acc: 2912.0/3175.0=0.9172\n",
      "Instance: 160; Time: 0.14s; loss: 112.1675; acc: 3232.0/3527.0=0.9164\n",
      "Instance: 180; Time: 0.18s; loss: 100.1654; acc: 3540.0/3869.0=0.9150\n",
      "Instance: 200; Time: 0.28s; loss: 137.1854; acc: 4046.0/4416.0=0.9162\n",
      "Instance: 220; Time: 0.20s; loss: 210.0060; acc: 4376.0/4810.0=0.9098\n",
      "Instance: 240; Time: 0.17s; loss: 111.1910; acc: 4829.0/5293.0=0.9123\n",
      "Instance: 260; Time: 0.16s; loss: 174.7055; acc: 5157.0/5668.0=0.9098\n",
      "Instance: 280; Time: 0.16s; loss: 114.1918; acc: 5551.0/6094.0=0.9109\n",
      "Instance: 300; Time: 0.19s; loss: 117.8931; acc: 5919.0/6493.0=0.9116\n",
      "Instance: 320; Time: 0.18s; loss: 119.0131; acc: 6359.0/6973.0=0.9119\n",
      "Instance: 340; Time: 0.17s; loss: 109.1547; acc: 6745.0/7390.0=0.9127\n",
      "Instance: 360; Time: 0.15s; loss: 126.5685; acc: 7120.0/7797.0=0.9132\n",
      "Instance: 380; Time: 0.20s; loss: 148.1708; acc: 7514.0/8237.0=0.9122\n",
      "Instance: 400; Time: 0.19s; loss: 147.4152; acc: 7950.0/8715.0=0.9122\n",
      "Instance: 420; Time: 0.16s; loss: 191.2814; acc: 8284.0/9101.0=0.9102\n",
      "Instance: 440; Time: 0.21s; loss: 146.2327; acc: 8754.0/9617.0=0.9103\n",
      "Instance: 460; Time: 0.30s; loss: 173.6425; acc: 9191.0/10109.0=0.9092\n",
      "Instance: 480; Time: 0.25s; loss: 104.9503; acc: 9536.0/10487.0=0.9093\n",
      "Instance: 500; Time: 0.26s; loss: 107.3818; acc: 10080.0/11062.0=0.9112\n",
      "     Instance: 501; Time: 0.06s; loss: 3.6321; acc: 10090.0/11074.0=0.9111\n",
      "Epoch: 5 training finished. Time: 4.92s, speed: 101.88st/s,  total loss: tensor(3398.7058, device='cuda:0')\n",
      "gold_num =  341  pred_num =  351  right_num =  261\n",
      "这是dev_loss： tensor(628.0645, device='cuda:0', grad_fn=<SubBackward0>)\n",
      "Dev: time: 0.68s, speed: 134.00st/s; acc: 0.8733, p: 0.7436, r: 0.7654, f: 0.7543\n",
      "gold_num =  697  pred_num =  706  right_num =  548\n",
      "Test: time: 1.13s, speed: 107.24st/s; acc: 0.8675, p: 0.7762, r: 0.7862, f: 0.7812\n",
      "Best dev score: p:0.7507082152974505, r:0.7771260997067448, f:0.7636887608069165\n",
      "Test score: p:0.7493036211699164, r:0.7718794835007173, f:0.7604240282685514\n",
      "Epoch: 6/16\n",
      " Learning rate is setted as: 0.003675459453124999\n",
      "Instance: 20; Time: 0.26s; loss: 116.5363; acc: 483.0/513.0=0.9415\n",
      "Instance: 40; Time: 0.16s; loss: 107.7139; acc: 815.0/876.0=0.9304\n",
      "Instance: 60; Time: 0.17s; loss: 163.3176; acc: 1203.0/1308.0=0.9197\n",
      "Instance: 80; Time: 0.21s; loss: 111.5518; acc: 1583.0/1729.0=0.9156\n",
      "Instance: 100; Time: 0.18s; loss: 63.0236; acc: 1915.0/2077.0=0.9220\n",
      "Instance: 120; Time: 0.18s; loss: 129.9618; acc: 2341.0/2529.0=0.9257\n",
      "Instance: 140; Time: 0.17s; loss: 119.6647; acc: 2660.0/2891.0=0.9201\n",
      "Instance: 160; Time: 0.15s; loss: 114.6779; acc: 3029.0/3302.0=0.9173\n",
      "Instance: 180; Time: 0.21s; loss: 163.4172; acc: 3435.0/3762.0=0.9131\n",
      "Instance: 200; Time: 0.15s; loss: 83.2117; acc: 3734.0/4090.0=0.9130\n",
      "Instance: 220; Time: 0.22s; loss: 121.6824; acc: 4189.0/4582.0=0.9142\n",
      "Instance: 240; Time: 0.22s; loss: 125.7733; acc: 4642.0/5071.0=0.9154\n",
      "Instance: 260; Time: 0.24s; loss: 125.9999; acc: 5080.0/5541.0=0.9168\n",
      "Instance: 280; Time: 0.17s; loss: 151.8187; acc: 5464.0/5973.0=0.9148\n",
      "Instance: 300; Time: 0.15s; loss: 99.8904; acc: 5820.0/6361.0=0.9150\n",
      "Instance: 320; Time: 0.20s; loss: 127.6163; acc: 6292.0/6873.0=0.9155\n",
      "Instance: 340; Time: 0.18s; loss: 96.4556; acc: 6716.0/7324.0=0.9170\n",
      "Instance: 360; Time: 0.24s; loss: 129.6783; acc: 7246.0/7890.0=0.9184\n",
      "Instance: 380; Time: 0.21s; loss: 136.9236; acc: 7627.0/8309.0=0.9179\n",
      "Instance: 400; Time: 0.17s; loss: 110.3364; acc: 7966.0/8682.0=0.9175\n",
      "Instance: 420; Time: 0.16s; loss: 111.2561; acc: 8423.0/9168.0=0.9187\n",
      "Instance: 440; Time: 0.15s; loss: 101.4781; acc: 8836.0/9613.0=0.9192\n",
      "Instance: 460; Time: 0.18s; loss: 138.7559; acc: 9269.0/10081.0=0.9195\n",
      "Instance: 480; Time: 0.29s; loss: 135.7715; acc: 9713.0/10571.0=0.9188\n",
      "Instance: 500; Time: 0.20s; loss: 111.5740; acc: 10159.0/11052.0=0.9192\n",
      "     Instance: 501; Time: 0.07s; loss: 4.5222; acc: 10180.0/11074.0=0.9193\n",
      "Epoch: 6 training finished. Time: 4.90s, speed: 102.21st/s,  total loss: tensor(3002.6086, device='cuda:0')\n",
      "gold_num =  341  pred_num =  350  right_num =  259\n",
      "这是dev_loss： tensor(607.8057, device='cuda:0', grad_fn=<SubBackward0>)\n",
      "Dev: time: 0.68s, speed: 135.70st/s; acc: 0.8707, p: 0.7400, r: 0.7595, f: 0.7496\n",
      "gold_num =  697  pred_num =  706  right_num =  553\n",
      "Test: time: 1.12s, speed: 108.69st/s; acc: 0.8700, p: 0.7833, r: 0.7934, f: 0.7883\n",
      "Best dev score: p:0.7507082152974505, r:0.7771260997067448, f:0.7636887608069165\n",
      "Test score: p:0.7493036211699164, r:0.7718794835007173, f:0.7604240282685514\n",
      "Epoch: 7/16\n",
      " Learning rate is setted as: 0.003491686480468749\n",
      "Instance: 20; Time: 0.15s; loss: 110.7544; acc: 487.0/516.0=0.9438\n",
      "Instance: 40; Time: 0.16s; loss: 179.7452; acc: 789.0/870.0=0.9069\n",
      "Instance: 60; Time: 0.18s; loss: 116.5350; acc: 1215.0/1324.0=0.9177\n",
      "Instance: 80; Time: 0.20s; loss: 149.2164; acc: 1643.0/1792.0=0.9169\n",
      "Instance: 100; Time: 0.21s; loss: 109.5613; acc: 2098.0/2282.0=0.9194\n",
      "Instance: 120; Time: 0.21s; loss: 83.3767; acc: 2504.0/2718.0=0.9213\n",
      "Instance: 140; Time: 0.19s; loss: 109.8833; acc: 2902.0/3143.0=0.9233\n",
      "Instance: 160; Time: 0.15s; loss: 84.7819; acc: 3272.0/3539.0=0.9246\n",
      "Instance: 180; Time: 0.16s; loss: 112.9974; acc: 3627.0/3924.0=0.9243\n",
      "Instance: 200; Time: 0.15s; loss: 123.2056; acc: 4004.0/4335.0=0.9236\n",
      "Instance: 220; Time: 0.18s; loss: 113.0374; acc: 4404.0/4769.0=0.9235\n",
      "Instance: 240; Time: 0.17s; loss: 95.8774; acc: 4717.0/5112.0=0.9227\n",
      "Instance: 260; Time: 0.11s; loss: 83.0893; acc: 4998.0/5423.0=0.9216\n",
      "Instance: 280; Time: 0.21s; loss: 106.3696; acc: 5459.0/5914.0=0.9231\n",
      "Instance: 300; Time: 0.26s; loss: 97.5254; acc: 5884.0/6373.0=0.9233\n",
      "Instance: 320; Time: 0.16s; loss: 76.9276; acc: 6230.0/6743.0=0.9239\n",
      "Instance: 340; Time: 0.20s; loss: 113.9054; acc: 6612.0/7170.0=0.9222\n",
      "Instance: 360; Time: 0.25s; loss: 124.7688; acc: 7102.0/7701.0=0.9222\n",
      "Instance: 380; Time: 0.23s; loss: 122.8845; acc: 7572.0/8204.0=0.9230\n",
      "Instance: 400; Time: 0.24s; loss: 122.6597; acc: 8033.0/8700.0=0.9233\n",
      "Instance: 420; Time: 0.22s; loss: 123.1193; acc: 8469.0/9172.0=0.9234\n",
      "Instance: 440; Time: 0.27s; loss: 115.5171; acc: 8963.0/9702.0=0.9238\n",
      "Instance: 460; Time: 0.18s; loss: 144.5800; acc: 9327.0/10116.0=0.9220\n",
      "Instance: 480; Time: 0.17s; loss: 111.4423; acc: 9770.0/10593.0=0.9223\n",
      "Instance: 500; Time: 0.29s; loss: 102.0748; acc: 10199.0/11051.0=0.9229\n",
      "     Instance: 501; Time: 0.07s; loss: 1.1094; acc: 10222.0/11074.0=0.9231\n",
      "Epoch: 7 training finished. Time: 4.95s, speed: 101.19st/s,  total loss: tensor(2834.9453, device='cuda:0')\n",
      "gold_num =  341  pred_num =  348  right_num =  267\n",
      "这是dev_loss： tensor(606.5215, device='cuda:0', grad_fn=<SubBackward0>)\n",
      "Dev: time: 0.67s, speed: 136.79st/s; acc: 0.8790, p: 0.7672, r: 0.7830, f: 0.7750\n",
      "Exceed previous best f score: 0.7636887608069165\n",
      "gold_num =  697  pred_num =  704  right_num =  544\n",
      "Test: time: 1.12s, speed: 111.14st/s; acc: 0.8662, p: 0.7727, r: 0.7805, f: 0.7766\n",
      "Best dev score: p:0.7672413793103449, r:0.782991202346041, f:0.7750362844702466\n",
      "Test score: p:0.7727272727272727, r:0.7804878048780488, f:0.7765881513204854\n",
      "Epoch: 8/16\n",
      " Learning rate is setted as: 0.0033171021564453113\n",
      "Instance: 20; Time: 0.12s; loss: 61.7427; acc: 349.0/363.0=0.9614\n",
      "Instance: 40; Time: 0.18s; loss: 74.5809; acc: 771.0/805.0=0.9578\n",
      "Instance: 60; Time: 0.20s; loss: 102.8105; acc: 1213.0/1273.0=0.9529\n",
      "Instance: 80; Time: 0.26s; loss: 94.6095; acc: 1633.0/1722.0=0.9483\n",
      "Instance: 100; Time: 0.19s; loss: 74.2787; acc: 2074.0/2187.0=0.9483\n",
      "Instance: 120; Time: 0.20s; loss: 111.3811; acc: 2512.0/2653.0=0.9469\n",
      "Instance: 140; Time: 0.26s; loss: 86.0026; acc: 2974.0/3138.0=0.9477\n",
      "Instance: 160; Time: 0.21s; loss: 114.8103; acc: 3490.0/3684.0=0.9473\n",
      "Instance: 180; Time: 0.18s; loss: 103.4205; acc: 3918.0/4144.0=0.9455\n",
      "Instance: 200; Time: 0.23s; loss: 72.8894; acc: 4281.0/4534.0=0.9442\n",
      "Instance: 220; Time: 0.15s; loss: 92.8804; acc: 4616.0/4895.0=0.9430\n",
      "Instance: 240; Time: 0.14s; loss: 140.6619; acc: 4945.0/5269.0=0.9385\n",
      "Instance: 260; Time: 0.17s; loss: 70.7964; acc: 5303.0/5649.0=0.9388\n",
      "Instance: 280; Time: 0.13s; loss: 110.1544; acc: 5655.0/6034.0=0.9372\n",
      "Instance: 300; Time: 0.25s; loss: 68.8936; acc: 6115.0/6508.0=0.9396\n",
      "Instance: 320; Time: 0.16s; loss: 147.3162; acc: 6447.0/6889.0=0.9358\n",
      "Instance: 340; Time: 0.22s; loss: 123.7615; acc: 6887.0/7367.0=0.9348\n",
      "Instance: 360; Time: 0.24s; loss: 97.6641; acc: 7263.0/7773.0=0.9344\n",
      "Instance: 380; Time: 0.26s; loss: 133.2963; acc: 7675.0/8226.0=0.9330\n",
      "Instance: 400; Time: 0.25s; loss: 167.7760; acc: 8163.0/8776.0=0.9302\n",
      "Instance: 420; Time: 0.17s; loss: 138.7528; acc: 8575.0/9229.0=0.9291\n",
      "Instance: 440; Time: 0.16s; loss: 86.5498; acc: 8915.0/9594.0=0.9292\n",
      "Instance: 460; Time: 0.15s; loss: 107.9636; acc: 9283.0/10000.0=0.9283\n",
      "Instance: 480; Time: 0.23s; loss: 124.5496; acc: 9802.0/10555.0=0.9287\n",
      "Instance: 500; Time: 0.19s; loss: 82.9031; acc: 10283.0/11058.0=0.9299\n",
      "     Instance: 501; Time: 0.09s; loss: 1.6203; acc: 10299.0/11074.0=0.9300\n",
      "Epoch: 8 training finished. Time: 4.97s, speed: 100.73st/s,  total loss: tensor(2592.0659, device='cuda:0')\n",
      "gold_num =  341  pred_num =  351  right_num =  267\n",
      "这是dev_loss： tensor(607.5557, device='cuda:0', grad_fn=<SubBackward0>)\n",
      "Dev: time: 0.68s, speed: 135.91st/s; acc: 0.8758, p: 0.7607, r: 0.7830, f: 0.7717\n",
      "gold_num =  697  pred_num =  726  right_num =  557\n",
      "Test: time: 1.12s, speed: 107.38st/s; acc: 0.8684, p: 0.7672, r: 0.7991, f: 0.7829\n",
      "Best dev score: p:0.7672413793103449, r:0.782991202346041, f:0.7750362844702466\n",
      "Test score: p:0.7727272727272727, r:0.7804878048780488, f:0.7765881513204854\n",
      "Epoch: 9/16\n",
      " Learning rate is setted as: 0.0031512470486230455\n",
      "Instance: 20; Time: 0.14s; loss: 95.0646; acc: 403.0/428.0=0.9416\n",
      "Instance: 40; Time: 0.18s; loss: 125.0430; acc: 838.0/906.0=0.9249\n",
      "Instance: 60; Time: 0.34s; loss: 84.9865; acc: 1386.0/1472.0=0.9416\n",
      "Instance: 80; Time: 0.25s; loss: 93.7804; acc: 1911.0/2026.0=0.9432\n",
      "Instance: 100; Time: 0.18s; loss: 79.6328; acc: 2349.0/2483.0=0.9460\n",
      "Instance: 120; Time: 0.18s; loss: 76.3872; acc: 2665.0/2824.0=0.9437\n",
      "Instance: 140; Time: 0.16s; loss: 130.2554; acc: 3065.0/3264.0=0.9390\n",
      "Instance: 160; Time: 0.19s; loss: 125.5571; acc: 3465.0/3701.0=0.9362\n",
      "Instance: 180; Time: 0.21s; loss: 77.5454; acc: 3899.0/4158.0=0.9377\n",
      "Instance: 200; Time: 0.20s; loss: 89.8348; acc: 4328.0/4611.0=0.9386\n",
      "Instance: 220; Time: 0.20s; loss: 95.5568; acc: 4751.0/5066.0=0.9378\n",
      "Instance: 240; Time: 0.24s; loss: 74.7682; acc: 5230.0/5562.0=0.9403\n",
      "Instance: 260; Time: 0.28s; loss: 165.8661; acc: 5735.0/6117.0=0.9376\n",
      "Instance: 280; Time: 0.22s; loss: 100.5178; acc: 6111.0/6524.0=0.9367\n",
      "Instance: 300; Time: 0.17s; loss: 91.2975; acc: 6468.0/6903.0=0.9370\n",
      "Instance: 320; Time: 0.18s; loss: 80.1693; acc: 6919.0/7374.0=0.9383\n",
      "Instance: 340; Time: 0.20s; loss: 63.5109; acc: 7236.0/7716.0=0.9378\n",
      "Instance: 360; Time: 0.17s; loss: 88.3978; acc: 7608.0/8114.0=0.9376\n",
      "Instance: 380; Time: 0.17s; loss: 119.4415; acc: 8032.0/8573.0=0.9369\n",
      "Instance: 400; Time: 0.19s; loss: 100.5940; acc: 8364.0/8936.0=0.9360\n",
      "Instance: 420; Time: 0.16s; loss: 96.2542; acc: 8664.0/9266.0=0.9350\n",
      "Instance: 440; Time: 0.15s; loss: 102.2009; acc: 9019.0/9657.0=0.9339\n",
      "Instance: 460; Time: 0.21s; loss: 92.1930; acc: 9443.0/10107.0=0.9343\n",
      "Instance: 480; Time: 0.23s; loss: 57.2537; acc: 9854.0/10533.0=0.9355\n",
      "Instance: 500; Time: 0.24s; loss: 153.9539; acc: 10293.0/11021.0=0.9339\n",
      "     Instance: 501; Time: 0.13s; loss: 1.5915; acc: 10346.0/11074.0=0.9343\n",
      "Epoch: 9 training finished. Time: 5.14s, speed: 97.56st/s,  total loss: tensor(2461.6541, device='cuda:0')\n",
      "gold_num =  341  pred_num =  352  right_num =  267\n",
      "这是dev_loss： tensor(597.7734, device='cuda:0', grad_fn=<SubBackward0>)\n",
      "Dev: time: 0.68s, speed: 134.63st/s; acc: 0.8752, p: 0.7585, r: 0.7830, f: 0.7706\n",
      "gold_num =  697  pred_num =  714  right_num =  556\n",
      "Test: time: 1.11s, speed: 108.88st/s; acc: 0.8697, p: 0.7787, r: 0.7977, f: 0.7881\n",
      "Best dev score: p:0.7672413793103449, r:0.782991202346041, f:0.7750362844702466\n",
      "Test score: p:0.7727272727272727, r:0.7804878048780488, f:0.7765881513204854\n",
      "Epoch: 10/16\n",
      " Learning rate is setted as: 0.0029936846961918936\n",
      "Instance: 20; Time: 0.18s; loss: 116.8397; acc: 506.0/542.0=0.9336\n",
      "Instance: 40; Time: 0.21s; loss: 85.8344; acc: 965.0/1032.0=0.9351\n",
      "Instance: 60; Time: 0.16s; loss: 98.1142; acc: 1284.0/1379.0=0.9311\n",
      "Instance: 80; Time: 0.19s; loss: 105.5431; acc: 1790.0/1916.0=0.9342\n",
      "Instance: 100; Time: 0.19s; loss: 100.4957; acc: 2216.0/2380.0=0.9311\n",
      "Instance: 120; Time: 0.21s; loss: 78.8572; acc: 2682.0/2868.0=0.9351\n",
      "Instance: 140; Time: 0.28s; loss: 86.3292; acc: 3170.0/3376.0=0.9390\n",
      "Instance: 160; Time: 0.22s; loss: 87.8751; acc: 3675.0/3907.0=0.9406\n",
      "Instance: 180; Time: 0.15s; loss: 81.4413; acc: 3946.0/4204.0=0.9386\n",
      "Instance: 200; Time: 0.12s; loss: 74.3665; acc: 4309.0/4587.0=0.9394\n",
      "Instance: 220; Time: 0.20s; loss: 69.3954; acc: 4737.0/5039.0=0.9401\n",
      "Instance: 240; Time: 0.18s; loss: 74.6268; acc: 5104.0/5432.0=0.9396\n",
      "Instance: 260; Time: 0.22s; loss: 105.8944; acc: 5533.0/5890.0=0.9394\n",
      "Instance: 280; Time: 0.16s; loss: 62.4468; acc: 5835.0/6216.0=0.9387\n",
      "Instance: 300; Time: 0.14s; loss: 76.4136; acc: 6210.0/6617.0=0.9385\n",
      "Instance: 320; Time: 0.17s; loss: 139.2549; acc: 6722.0/7167.0=0.9379\n",
      "Instance: 340; Time: 0.18s; loss: 108.0339; acc: 7071.0/7550.0=0.9366\n",
      "Instance: 360; Time: 0.15s; loss: 72.2297; acc: 7430.0/7926.0=0.9374\n",
      "Instance: 380; Time: 0.15s; loss: 80.1278; acc: 7819.0/8338.0=0.9378\n",
      "Instance: 400; Time: 0.21s; loss: 84.6207; acc: 8290.0/8832.0=0.9386\n",
      "Instance: 420; Time: 0.21s; loss: 70.6321; acc: 8676.0/9242.0=0.9388\n",
      "Instance: 440; Time: 0.19s; loss: 81.7422; acc: 9117.0/9709.0=0.9390\n",
      "Instance: 460; Time: 0.22s; loss: 113.3611; acc: 9596.0/10230.0=0.9380\n",
      "Instance: 480; Time: 0.20s; loss: 127.2310; acc: 9993.0/10672.0=0.9364\n",
      "Instance: 500; Time: 0.18s; loss: 63.3040; acc: 10362.0/11057.0=0.9371\n",
      "     Instance: 501; Time: 0.06s; loss: 0.9999; acc: 10379.0/11074.0=0.9372\n",
      "Epoch: 10 training finished. Time: 4.73s, speed: 105.97st/s,  total loss: tensor(2246.0107, device='cuda:0')\n",
      "gold_num =  341  pred_num =  356  right_num =  270\n",
      "这是dev_loss： tensor(607.1953, device='cuda:0', grad_fn=<SubBackward0>)\n",
      "Dev: time: 0.67s, speed: 134.90st/s; acc: 0.8802, p: 0.7584, r: 0.7918, f: 0.7747\n",
      "gold_num =  697  pred_num =  713  right_num =  560\n",
      "Test: time: 1.06s, speed: 113.51st/s; acc: 0.8716, p: 0.7854, r: 0.8034, f: 0.7943\n",
      "Best dev score: p:0.7672413793103449, r:0.782991202346041, f:0.7750362844702466\n",
      "Test score: p:0.7727272727272727, r:0.7804878048780488, f:0.7765881513204854\n",
      "Epoch: 11/16\n",
      " Learning rate is setted as: 0.0028440004613822984\n",
      "Instance: 20; Time: 0.13s; loss: 66.9882; acc: 369.0/387.0=0.9535\n",
      "Instance: 40; Time: 0.24s; loss: 74.9664; acc: 803.0/838.0=0.9582\n",
      "Instance: 60; Time: 0.22s; loss: 102.6820; acc: 1216.0/1279.0=0.9507\n",
      "Instance: 80; Time: 0.15s; loss: 63.5493; acc: 1608.0/1693.0=0.9498\n",
      "Instance: 100; Time: 0.21s; loss: 110.9144; acc: 2032.0/2153.0=0.9438\n",
      "Instance: 120; Time: 0.14s; loss: 72.4346; acc: 2336.0/2480.0=0.9419\n",
      "Instance: 140; Time: 0.21s; loss: 114.7412; acc: 2904.0/3078.0=0.9435\n",
      "Instance: 160; Time: 0.22s; loss: 61.6086; acc: 3290.0/3479.0=0.9457\n",
      "Instance: 180; Time: 0.16s; loss: 71.5745; acc: 3671.0/3882.0=0.9456\n",
      "Instance: 200; Time: 0.19s; loss: 75.0327; acc: 4118.0/4351.0=0.9464\n",
      "Instance: 220; Time: 0.18s; loss: 105.0790; acc: 4588.0/4844.0=0.9472\n",
      "Instance: 240; Time: 0.19s; loss: 63.3173; acc: 4961.0/5232.0=0.9482\n",
      "Instance: 260; Time: 0.22s; loss: 119.6473; acc: 5370.0/5679.0=0.9456\n",
      "Instance: 280; Time: 0.14s; loss: 93.3772; acc: 5741.0/6086.0=0.9433\n",
      "Instance: 300; Time: 0.24s; loss: 109.6423; acc: 6200.0/6578.0=0.9425\n",
      "Instance: 320; Time: 0.29s; loss: 110.7258; acc: 6670.0/7087.0=0.9412\n",
      "Instance: 340; Time: 0.18s; loss: 72.3110; acc: 7012.0/7452.0=0.9410\n",
      "Instance: 360; Time: 0.15s; loss: 75.5948; acc: 7381.0/7849.0=0.9404\n",
      "Instance: 380; Time: 0.22s; loss: 96.0104; acc: 7863.0/8359.0=0.9407\n",
      "Instance: 400; Time: 0.25s; loss: 121.1133; acc: 8351.0/8889.0=0.9395\n",
      "Instance: 420; Time: 0.22s; loss: 99.6261; acc: 8795.0/9361.0=0.9395\n",
      "Instance: 440; Time: 0.22s; loss: 74.0992; acc: 9136.0/9726.0=0.9393\n",
      "Instance: 460; Time: 0.21s; loss: 49.2661; acc: 9576.0/10180.0=0.9407\n",
      "Instance: 480; Time: 0.18s; loss: 94.2244; acc: 9985.0/10625.0=0.9398\n",
      "Instance: 500; Time: 0.16s; loss: 76.0867; acc: 10386.0/11052.0=0.9397\n",
      "     Instance: 501; Time: 0.06s; loss: 2.4335; acc: 10407.0/11074.0=0.9398\n",
      "Epoch: 11 training finished. Time: 4.99s, speed: 100.39st/s,  total loss: tensor(2177.0466, device='cuda:0')\n",
      "gold_num =  341  pred_num =  353  right_num =  270\n",
      "这是dev_loss： tensor(577.6982, device='cuda:0', grad_fn=<SubBackward0>)\n",
      "Dev: time: 0.69s, speed: 131.93st/s; acc: 0.8809, p: 0.7649, r: 0.7918, f: 0.7781\n",
      "Exceed previous best f score: 0.7750362844702466\n",
      "gold_num =  697  pred_num =  711  right_num =  555\n",
      "Test: time: 1.10s, speed: 113.06st/s; acc: 0.8681, p: 0.7806, r: 0.7963, f: 0.7884\n",
      "Best dev score: p:0.7648725212464589, r:0.7917888563049853, f:0.7780979827089336\n",
      "Test score: p:0.7805907172995781, r:0.7962697274031564, f:0.7883522727272727\n",
      "Epoch: 12/16\n",
      " Learning rate is setted as: 0.0027018004383131834\n",
      "Instance: 20; Time: 0.17s; loss: 99.3267; acc: 550.0/587.0=0.9370\n",
      "Instance: 40; Time: 0.18s; loss: 89.9249; acc: 984.0/1044.0=0.9425\n",
      "Instance: 60; Time: 0.22s; loss: 75.2052; acc: 1436.0/1516.0=0.9472\n",
      "Instance: 80; Time: 0.24s; loss: 64.0010; acc: 1884.0/1981.0=0.9510\n",
      "Instance: 100; Time: 0.15s; loss: 97.6934; acc: 2234.0/2361.0=0.9462\n",
      "Instance: 120; Time: 0.17s; loss: 99.7516; acc: 2600.0/2764.0=0.9407\n",
      "Instance: 140; Time: 0.27s; loss: 121.6453; acc: 3232.0/3435.0=0.9409\n",
      "Instance: 160; Time: 0.25s; loss: 46.9559; acc: 3602.0/3814.0=0.9444\n",
      "Instance: 180; Time: 0.13s; loss: 91.4480; acc: 3971.0/4211.0=0.9430\n",
      "Instance: 200; Time: 0.17s; loss: 73.6702; acc: 4422.0/4682.0=0.9445\n",
      "Instance: 220; Time: 0.29s; loss: 128.8514; acc: 4925.0/5231.0=0.9415\n",
      "Instance: 240; Time: 0.18s; loss: 83.5142; acc: 5271.0/5607.0=0.9401\n",
      "Instance: 260; Time: 0.17s; loss: 62.8940; acc: 5608.0/5964.0=0.9403\n",
      "Instance: 280; Time: 0.16s; loss: 102.7694; acc: 5946.0/6340.0=0.9379\n",
      "Instance: 300; Time: 0.27s; loss: 97.2054; acc: 6432.0/6849.0=0.9391\n",
      "Instance: 320; Time: 0.21s; loss: 66.9570; acc: 6881.0/7321.0=0.9399\n",
      "Instance: 340; Time: 0.18s; loss: 71.7831; acc: 7190.0/7653.0=0.9395\n",
      "Instance: 360; Time: 0.24s; loss: 79.2316; acc: 7667.0/8156.0=0.9400\n",
      "Instance: 380; Time: 0.20s; loss: 82.5693; acc: 8055.0/8574.0=0.9395\n",
      "Instance: 400; Time: 0.15s; loss: 78.8351; acc: 8415.0/8960.0=0.9392\n",
      "Instance: 420; Time: 0.16s; loss: 47.2457; acc: 8709.0/9265.0=0.9400\n",
      "Instance: 440; Time: 0.23s; loss: 86.7858; acc: 9126.0/9705.0=0.9403\n",
      "Instance: 460; Time: 0.14s; loss: 49.4038; acc: 9461.0/10054.0=0.9410\n",
      "Instance: 480; Time: 0.19s; loss: 70.7771; acc: 9958.0/10573.0=0.9418\n",
      "Instance: 500; Time: 0.17s; loss: 63.2645; acc: 10434.0/11067.0=0.9428\n",
      "     Instance: 501; Time: 0.07s; loss: 0.6781; acc: 10441.0/11074.0=0.9428\n",
      "Epoch: 12 training finished. Time: 4.96s, speed: 100.92st/s,  total loss: tensor(2032.3877, device='cuda:0')\n",
      "gold_num =  341  pred_num =  354  right_num =  272\n",
      "这是dev_loss： tensor(609.0234, device='cuda:0', grad_fn=<SubBackward0>)\n",
      "Dev: time: 0.71s, speed: 130.46st/s; acc: 0.8777, p: 0.7684, r: 0.7977, f: 0.7827\n",
      "Exceed previous best f score: 0.7780979827089336\n",
      "gold_num =  697  pred_num =  723  right_num =  560\n",
      "Test: time: 1.16s, speed: 106.64st/s; acc: 0.8697, p: 0.7746, r: 0.8034, f: 0.7887\n",
      "Best dev score: p:0.768361581920904, r:0.7976539589442815, f:0.7827338129496403\n",
      "Test score: p:0.7745504840940526, r:0.8034433285509326, f:0.7887323943661971\n",
      "Epoch: 13/16\n",
      " Learning rate is setted as: 0.002566710416397524\n",
      "Instance: 20; Time: 0.12s; loss: 53.1473; acc: 326.0/339.0=0.9617\n",
      "Instance: 40; Time: 0.24s; loss: 102.4856; acc: 910.0/949.0=0.9589\n",
      "Instance: 60; Time: 0.23s; loss: 72.9276; acc: 1373.0/1431.0=0.9595\n",
      "Instance: 80; Time: 0.18s; loss: 84.8815; acc: 1747.0/1833.0=0.9531\n",
      "Instance: 100; Time: 0.24s; loss: 69.5458; acc: 2162.0/2277.0=0.9495\n",
      "Instance: 120; Time: 0.21s; loss: 99.3940; acc: 2596.0/2743.0=0.9464\n",
      "Instance: 140; Time: 0.25s; loss: 106.4393; acc: 3074.0/3255.0=0.9444\n",
      "Instance: 160; Time: 0.24s; loss: 66.6213; acc: 3447.0/3656.0=0.9428\n",
      "Instance: 180; Time: 0.19s; loss: 80.9860; acc: 3897.0/4134.0=0.9427\n",
      "Instance: 200; Time: 0.19s; loss: 74.3607; acc: 4364.0/4621.0=0.9444\n",
      "Instance: 220; Time: 0.21s; loss: 73.2225; acc: 4776.0/5053.0=0.9452\n",
      "Instance: 240; Time: 0.19s; loss: 41.8683; acc: 5232.0/5514.0=0.9489\n",
      "Instance: 260; Time: 0.14s; loss: 55.9784; acc: 5576.0/5871.0=0.9498\n",
      "Instance: 280; Time: 0.15s; loss: 81.6552; acc: 5900.0/6223.0=0.9481\n",
      "Instance: 300; Time: 0.14s; loss: 53.0295; acc: 6319.0/6652.0=0.9499\n",
      "Instance: 320; Time: 0.19s; loss: 83.7765; acc: 6722.0/7086.0=0.9486\n",
      "Instance: 340; Time: 0.18s; loss: 51.7037; acc: 7122.0/7504.0=0.9491\n",
      "Instance: 360; Time: 0.28s; loss: 64.3579; acc: 7518.0/7922.0=0.9490\n",
      "Instance: 380; Time: 0.17s; loss: 72.1727; acc: 7895.0/8330.0=0.9478\n",
      "Instance: 400; Time: 0.19s; loss: 121.9646; acc: 8320.0/8786.0=0.9470\n",
      "Instance: 420; Time: 0.17s; loss: 77.4789; acc: 8685.0/9176.0=0.9465\n",
      "Instance: 440; Time: 0.24s; loss: 56.5609; acc: 9047.0/9553.0=0.9470\n",
      "Instance: 460; Time: 0.22s; loss: 100.7393; acc: 9545.0/10087.0=0.9463\n",
      "Instance: 480; Time: 0.28s; loss: 113.3003; acc: 9981.0/10567.0=0.9445\n",
      "Instance: 500; Time: 0.23s; loss: 83.3248; acc: 10441.0/11060.0=0.9440\n",
      "     Instance: 501; Time: 0.10s; loss: 5.7006; acc: 10452.0/11074.0=0.9438\n",
      "Epoch: 13 training finished. Time: 5.19s, speed: 96.50st/s,  total loss: tensor(1947.6234, device='cuda:0')\n",
      "gold_num =  341  pred_num =  353  right_num =  273\n",
      "这是dev_loss： tensor(604.4873, device='cuda:0', grad_fn=<SubBackward0>)\n",
      "Dev: time: 0.68s, speed: 134.10st/s; acc: 0.8790, p: 0.7734, r: 0.8006, f: 0.7867\n",
      "Exceed previous best f score: 0.7827338129496403\n",
      "gold_num =  697  pred_num =  726  right_num =  562\n",
      "Test: time: 1.09s, speed: 113.40st/s; acc: 0.8694, p: 0.7741, r: 0.8063, f: 0.7899\n",
      "Best dev score: p:0.773371104815864, r:0.8005865102639296, f:0.7867435158501441\n",
      "Test score: p:0.7741046831955923, r:0.806312769010043, f:0.7898805340829235\n",
      "Epoch: 14/16\n",
      " Learning rate is setted as: 0.0024383748955776477\n",
      "Instance: 20; Time: 0.19s; loss: 77.1288; acc: 442.0/468.0=0.9444\n",
      "Instance: 40; Time: 0.13s; loss: 84.1768; acc: 793.0/843.0=0.9407\n",
      "Instance: 60; Time: 0.14s; loss: 66.5416; acc: 1149.0/1221.0=0.9410\n",
      "Instance: 80; Time: 0.12s; loss: 58.7054; acc: 1408.0/1504.0=0.9362\n",
      "Instance: 100; Time: 0.19s; loss: 52.7085; acc: 1923.0/2031.0=0.9468\n",
      "Instance: 120; Time: 0.27s; loss: 77.4102; acc: 2433.0/2565.0=0.9485\n",
      "Instance: 140; Time: 0.24s; loss: 80.1093; acc: 2881.0/3038.0=0.9483\n",
      "Instance: 160; Time: 0.25s; loss: 65.5905; acc: 3386.0/3566.0=0.9495\n",
      "Instance: 180; Time: 0.19s; loss: 69.7156; acc: 3669.0/3879.0=0.9459\n",
      "Instance: 200; Time: 0.27s; loss: 96.4592; acc: 4207.0/4450.0=0.9454\n",
      "Instance: 220; Time: 0.17s; loss: 62.9985; acc: 4617.0/4879.0=0.9463\n",
      "Instance: 240; Time: 0.15s; loss: 57.1942; acc: 4960.0/5237.0=0.9471\n",
      "Instance: 260; Time: 0.17s; loss: 51.2142; acc: 5352.0/5644.0=0.9483\n",
      "Instance: 280; Time: 0.16s; loss: 52.9641; acc: 5791.0/6096.0=0.9500\n",
      "Instance: 300; Time: 0.16s; loss: 97.1445; acc: 6153.0/6496.0=0.9472\n",
      "Instance: 320; Time: 0.18s; loss: 86.7368; acc: 6528.0/6910.0=0.9447\n",
      "Instance: 340; Time: 0.22s; loss: 47.6071; acc: 6899.0/7298.0=0.9453\n",
      "Instance: 360; Time: 0.18s; loss: 120.6891; acc: 7342.0/7775.0=0.9443\n",
      "Instance: 380; Time: 0.26s; loss: 106.6309; acc: 7849.0/8309.0=0.9446\n",
      "Instance: 400; Time: 0.20s; loss: 67.4078; acc: 8299.0/8780.0=0.9452\n",
      "Instance: 420; Time: 0.24s; loss: 77.5133; acc: 8660.0/9168.0=0.9446\n",
      "Instance: 440; Time: 0.25s; loss: 77.1646; acc: 9143.0/9673.0=0.9452\n",
      "Instance: 460; Time: 0.17s; loss: 69.2031; acc: 9548.0/10097.0=0.9456\n",
      "Instance: 480; Time: 0.17s; loss: 62.9928; acc: 9965.0/10541.0=0.9454\n",
      "Instance: 500; Time: 0.27s; loss: 77.9052; acc: 10441.0/11038.0=0.9459\n",
      "     Instance: 501; Time: 0.07s; loss: 2.6349; acc: 10477.0/11074.0=0.9461\n",
      "Epoch: 14 training finished. Time: 5.01s, speed: 100.08st/s,  total loss: tensor(1846.5469, device='cuda:0')\n",
      "gold_num =  341  pred_num =  348  right_num =  274\n",
      "这是dev_loss： tensor(584.0195, device='cuda:0', grad_fn=<SubBackward0>)\n",
      "Dev: time: 0.67s, speed: 135.09st/s; acc: 0.8859, p: 0.7874, r: 0.8035, f: 0.7954\n",
      "Exceed previous best f score: 0.7867435158501441\n",
      "gold_num =  697  pred_num =  713  right_num =  559\n",
      "Test: time: 1.14s, speed: 108.81st/s; acc: 0.8706, p: 0.7840, r: 0.8020, f: 0.7929\n",
      "Best dev score: p:0.7873563218390804, r:0.8035190615835777, f:0.7953555878084179\n",
      "Test score: p:0.7840112201963534, r:0.8020086083213773, f:0.7929078014184396\n",
      "Epoch: 15/16\n",
      " Learning rate is setted as: 0.002316456150798765\n",
      "Instance: 20; Time: 0.17s; loss: 80.1661; acc: 515.0/538.0=0.9572\n",
      "Instance: 40; Time: 0.21s; loss: 65.7736; acc: 945.0/990.0=0.9545\n",
      "Instance: 60; Time: 0.19s; loss: 56.2196; acc: 1354.0/1416.0=0.9562\n",
      "Instance: 80; Time: 0.17s; loss: 92.0166; acc: 1826.0/1914.0=0.9540\n",
      "Instance: 100; Time: 0.18s; loss: 53.0330; acc: 2247.0/2344.0=0.9586\n",
      "Instance: 120; Time: 0.16s; loss: 78.5935; acc: 2664.0/2784.0=0.9569\n",
      "Instance: 140; Time: 0.15s; loss: 68.4299; acc: 2997.0/3136.0=0.9557\n",
      "Instance: 160; Time: 0.15s; loss: 97.2499; acc: 3358.0/3526.0=0.9524\n",
      "Instance: 180; Time: 0.14s; loss: 63.4265; acc: 3682.0/3865.0=0.9527\n",
      "Instance: 200; Time: 0.21s; loss: 62.8163; acc: 4167.0/4372.0=0.9531\n",
      "Instance: 220; Time: 0.16s; loss: 117.5769; acc: 4566.0/4812.0=0.9489\n",
      "Instance: 240; Time: 0.32s; loss: 94.0640; acc: 5069.0/5347.0=0.9480\n",
      "Instance: 260; Time: 0.21s; loss: 92.7468; acc: 5472.0/5779.0=0.9469\n",
      "Instance: 280; Time: 0.14s; loss: 57.1854; acc: 5802.0/6126.0=0.9471\n",
      "Instance: 300; Time: 0.21s; loss: 75.6118; acc: 6227.0/6575.0=0.9471\n",
      "Instance: 320; Time: 0.20s; loss: 50.6212; acc: 6632.0/6996.0=0.9480\n",
      "Instance: 340; Time: 0.22s; loss: 77.1873; acc: 7166.0/7551.0=0.9490\n",
      "Instance: 360; Time: 0.19s; loss: 72.0846; acc: 7665.0/8073.0=0.9495\n",
      "Instance: 380; Time: 0.19s; loss: 47.2407; acc: 8091.0/8508.0=0.9510\n",
      "Instance: 400; Time: 0.22s; loss: 58.4143; acc: 8463.0/8896.0=0.9513\n",
      "Instance: 420; Time: 0.25s; loss: 85.0475; acc: 8948.0/9409.0=0.9510\n",
      "Instance: 440; Time: 0.19s; loss: 47.0424; acc: 9331.0/9805.0=0.9517\n",
      "Instance: 460; Time: 0.19s; loss: 72.5090; acc: 9678.0/10173.0=0.9513\n",
      "Instance: 480; Time: 0.24s; loss: 60.6366; acc: 10128.0/10646.0=0.9513\n",
      "Instance: 500; Time: 0.19s; loss: 56.3123; acc: 10526.0/11063.0=0.9515\n",
      "     Instance: 501; Time: 0.06s; loss: 4.6124; acc: 10536.0/11074.0=0.9514\n",
      "Epoch: 15 training finished. Time: 4.93s, speed: 101.66st/s,  total loss: tensor(1786.6180, device='cuda:0')\n",
      "gold_num =  341  pred_num =  348  right_num =  271\n",
      "这是dev_loss： tensor(609.7285, device='cuda:0', grad_fn=<SubBackward0>)\n",
      "Dev: time: 0.70s, speed: 130.27st/s; acc: 0.8783, p: 0.7787, r: 0.7947, f: 0.7866\n",
      "gold_num =  697  pred_num =  716  right_num =  563\n",
      "Test: time: 1.10s, speed: 110.17st/s; acc: 0.8706, p: 0.7863, r: 0.8077, f: 0.7969\n",
      "Best dev score: p:0.7873563218390804, r:0.8035190615835777, f:0.7953555878084179\n",
      "Test score: p:0.7840112201963534, r:0.8020086083213773, f:0.7929078014184396\n"
     ]
    }
   ],
   "source": [
    "if __name__ == '__main__':\n",
    "    parser = argparse.ArgumentParser()\n",
    "    parser.add_argument('--embedding',  help='Embedding for words', default='None')\n",
    "    parser.add_argument('--status', choices=['train', 'test'], help='update algorithm', default='train')\n",
    "    parser.add_argument('--modelpath', default=\"/content/drive/MyDrive/Colab Notebooks/LexionAN-master-master/save_model/LAB2/\")\n",
    "    parser.add_argument('--modelname', default=\"model\")\n",
    "    parser.add_argument('--savedset', help='Dir of saved data setting', default=\"/content/drive/MyDrive/Colab Notebooks/LexionAN-master-master/data/final_data-lab2.dset\")\n",
    "    parser.add_argument('--train', default=\"/content/drive/MyDrive/Colab Notebooks/LexionAN-master-master/data/data/LAB2/train (1).txt\")\n",
    "    parser.add_argument('--test', default=\"/content/drive/MyDrive/Colab Notebooks/LexionAN-master-master/data/data/LAB2/test (1).txt\" )\n",
    "    parser.add_argument('--dev', default=\"/content/drive/MyDrive/Colab Notebooks/LexionAN-master-master/data/data/LAB2/dev.txt\")\n",
    "    parser.add_argument('--seg', default=\"True\")\n",
    "    parser.add_argument('--extendalphabet', default=\"True\")\n",
    "    parser.add_argument('--raw')\n",
    "    parser.add_argument('--output')\n",
    "    parser.add_argument('--seed',default=511,type=int)\n",
    "    parser.add_argument('--labelcomment', default=\"\")\n",
    "    parser.add_argument('--resultfile',default=\"/content/drive/MyDrive/Colab Notebooks/LexionAN-master-master/result/result-lab2.txt\")\n",
    "    parser.add_argument('--num_iter',default=16,type=int)\n",
    "    parser.add_argument('--num_layer', default=2, type=int)\n",
    "    parser.add_argument('--lr', type=float, default=0.005)\n",
    "    parser.add_argument('--batch_size', type=int, default=10)\n",
    "    parser.add_argument('--hidden_dim', type=int, default=100)\n",
    "    parser.add_argument('--model_type', default='lstm')\n",
    "    parser.add_argument('--drop', type=float, default=0.5)\n",
    "\n",
    "    parser.add_argument('--use_biword', dest='use_biword', action='store_true', default=False)\n",
    "    # parser.set_defaults(use_biword=False)\n",
    "    parser.add_argument('--use_char', dest='use_char', action='store_true', default=False)\n",
    "    # parser.set_defaults(use_biword=False)\n",
    "    parser.add_argument('--use_count', action='store_true', default=True)\n",
    "    parser.add_argument('--use_bert', action='store_true', default=False)\n",
    "\n",
    "    args = parser.parse_args()\n",
    "\n",
    "    seed_num = args.seed\n",
    "    set_seed(seed_num)\n",
    "\n",
    "    train_file = args.train\n",
    "    dev_file = args.dev\n",
    "    test_file = args.test\n",
    "    raw_file = args.raw\n",
    "    # model_dir = args.loadmodel\n",
    "    output_file = args.output\n",
    "    if args.seg.lower() == \"true\":\n",
    "        seg = True\n",
    "    else:\n",
    "        seg = False\n",
    "    status = args.status.lower()\n",
    "\n",
    "    save_model_dir = args.modelpath+args.modelname\n",
    "    save_data_name = args.savedset\n",
    "    gpu = torch.cuda.is_available()\n",
    "\n",
    "    #char_emb = None\n",
    "    #bichar_emb = None\n",
    "    #gaz_file = None\n",
    "    char_emb = \"/content/drive/MyDrive/Colab Notebooks/LexionAN-master-master/data/sgns.sikuquanshu.vec\"\n",
    "    bichar_emb = \"/content/drive/MyDrive/Colab Notebooks/LexionAN-master-master/data/sgns.sikuquanshu.vec\"\n",
    "    gaz_file = \"/content/drive/MyDrive/Colab Notebooks/LexionAN-master-master/data/TCM_w2v.vec\"\n",
    "\n",
    "\n",
    "    print('hello world')\n",
    "    #args.use_biword=True\n",
    "    #args.use_char=True\n",
    "    sys.stdout.flush()\n",
    "\n",
    "    if status == 'train':\n",
    "        #if os.path.exists(save_data_name):\n",
    "        if False:\n",
    "            print('Loading processed data')\n",
    "            with open(save_data_name, 'rb') as fp:\n",
    "                data = pickle.load(fp)\n",
    "            data.HP_num_layer = args.num_layer\n",
    "            data.HP_batch_size = args.batch_size\n",
    "            data.HP_iteration = args.num_iter\n",
    "            data.label_comment = args.labelcomment\n",
    "            data.result_file = args.resultfile\n",
    "            data.HP_lr = args.lr\n",
    "            data.use_bigram = args.use_biword\n",
    "            data.HP_use_char = args.use_char\n",
    "            data.HP_hidden_dim = args.hidden_dim\n",
    "            data.HP_dropout = args.drop\n",
    "            data.HP_use_count = args.use_count\n",
    "            data.model_type = args.model_type\n",
    "            data.use_bert = args.use_bert\n",
    "        else:\n",
    "            data = Data()\n",
    "            data.HP_gpu = gpu\n",
    "            data.HP_use_char = args.use_char\n",
    "            data.HP_batch_size = args.batch_size\n",
    "            data.HP_num_layer = args.num_layer\n",
    "            data.HP_iteration = args.num_iter\n",
    "            data.use_bigram = args.use_biword\n",
    "            data.HP_dropout = args.drop\n",
    "            data.norm_gaz_emb = False\n",
    "            data.HP_fix_gaz_emb = False\n",
    "            data.label_comment = args.labelcomment\n",
    "            data.result_file = args.resultfile\n",
    "            data.HP_lr = args.lr\n",
    "            data.HP_hidden_dim = args.hidden_dim\n",
    "            data.HP_use_count = args.use_count\n",
    "            data.model_type = args.model_type\n",
    "            data.use_bert = args.use_bert\n",
    "            data_initialization(data, gaz_file, train_file, dev_file, test_file)\n",
    "            data.generate_instance_with_gaz(train_file,'train')\n",
    "            data.generate_instance_with_gaz(dev_file,'dev')\n",
    "            data.generate_instance_with_gaz(test_file,'test')\n",
    "            data.build_word_pretrain_emb(char_emb)\n",
    "            data.build_biword_pretrain_emb(bichar_emb)\n",
    "            data.build_gaz_pretrain_emb(gaz_file)\n",
    "\n",
    "            print('Dumping data')\n",
    "            with open(save_data_name, 'wb') as f:\n",
    "                pickle.dump(data, f)\n",
    "            set_seed(seed_num)\n",
    "        print('data.use_biword=',data.use_bigram)\n",
    "        model,train_log,dev_log,test_log = train(data, save_model_dir, seg)\n",
    "    elif status == 'test':\n",
    "        print('Loading processed data')\n",
    "        with open(save_data_name, 'rb') as fp:\n",
    "            data = pickle.load(fp)\n",
    "        data.HP_num_layer = args.num_layer\n",
    "        data.HP_iteration = args.num_iter\n",
    "        data.label_comment = args.labelcomment\n",
    "        data.result_file = args.resultfile\n",
    "        # data.HP_use_gaz = args.use_gaz\n",
    "        data.HP_lr = args.lr\n",
    "        data.use_bigram = args.use_biword\n",
    "        data.HP_use_char = args.use_char\n",
    "        data.model_type = args.model_type\n",
    "        data.HP_hidden_dim = args.hidden_dim\n",
    "        data.HP_use_count = args.use_count\n",
    "        data.generate_instance_with_gaz(test_file,'test')\n",
    "        load_model_decode(save_model_dir, data, 'test', gpu, seg)\n",
    "\n",
    "    else:\n",
    "        print( \"Invalid argument! Please use valid arguments! (train/test/decode)\")\n",
    "\n",
    "torch.save(model,\"/content/drive/MyDrive/Colab Notebooks/LexionAN-master-master/save_model/LAB2/final-lab2.model\")\n",
    "\n",
    "\n",
    "filename='/content/drive/MyDrive/Colab Notebooks/LexionAN-master-master/save_model/LAB2/test_log-lab2.json'\n",
    "with open(filename,'w',encoding='utf-8') as file_obj:\n",
    "    json.dump(test_log,file_obj,ensure_ascii=False,indent = 4)\n",
    "\n",
    "filename='/content/drive/MyDrive/Colab Notebooks/LexionAN-master-master/save_model/LAB2/train_log-lab2.json'\n",
    "with open(filename,'w',encoding='utf-8') as file_obj:\n",
    "    json.dump(train_log,file_obj,ensure_ascii=False,indent = 4)\n",
    "\n",
    "filename='/content/drive/MyDrive/Colab Notebooks/LexionAN-master-master/save_model/LAB2/dev_log-lab2.json'\n",
    "with open(filename,'w',encoding='utf-8') as file_obj:\n",
    "    json.dump(dev_log,file_obj,ensure_ascii=False,indent = 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "executionInfo": {
     "elapsed": 2217,
     "status": "ok",
     "timestamp": 1651149620220,
     "user": {
      "displayName": "YING XU",
      "userId": "11756690436182978895"
     },
     "user_tz": -480
    },
    "id": "9ihyD_9pQ3Ka"
   },
   "outputs": [],
   "source": [
    "with open(\"/content/drive/MyDrive/Colab Notebooks/LexionAN-master-master/data/final_data-lab2.dset\", 'rb') as fp:\n",
    "  data = pickle.load(fp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "7mY5avpcOhey"
   },
   "source": [
    "# 新段落"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "executionInfo": {
     "elapsed": 447,
     "status": "ok",
     "timestamp": 1651149623775,
     "user": {
      "displayName": "YING XU",
      "userId": "11756690436182978895"
     },
     "user_tz": -480
    },
    "id": "TlAtL_EjlHnz"
   },
   "outputs": [],
   "source": [
    "ls = []\n",
    "for i in data.test_texts:\n",
    "  ls.extend(i[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "executionInfo": {
     "elapsed": 1,
     "status": "ok",
     "timestamp": 1651149625420,
     "user": {
      "displayName": "YING XU",
      "userId": "11756690436182978895"
     },
     "user_tz": -480
    },
    "id": "TzlAqZ9iRpSb"
   },
   "outputs": [],
   "source": [
    "with open(\"/content/drive/MyDrive/Colab Notebooks/LexionAN-master-master/save_model/LAB2/test_gold-lab2.json\",'w',encoding='utf-8') as file_obj:\n",
    "    json.dump(ls,file_obj,ensure_ascii=False,indent = 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "executionInfo": {
     "elapsed": 448,
     "status": "ok",
     "timestamp": 1651149627808,
     "user": {
      "displayName": "YING XU",
      "userId": "11756690436182978895"
     },
     "user_tz": -480
    },
    "id": "tmVYHYxyrqdw",
    "outputId": "1015c41e-91ce-4336-ffc5-2b293eadd28e"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7fdcc8827b50>"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/dist-packages/matplotlib/backends/backend_agg.py:214: RuntimeWarning: Glyph 35757 missing from current font.\n",
      "  font.set_text(s, 0.0, flags=flags)\n",
      "/usr/local/lib/python3.7/dist-packages/matplotlib/backends/backend_agg.py:214: RuntimeWarning: Glyph 32451 missing from current font.\n",
      "  font.set_text(s, 0.0, flags=flags)\n",
      "/usr/local/lib/python3.7/dist-packages/matplotlib/backends/backend_agg.py:214: RuntimeWarning: Glyph 38598 missing from current font.\n",
      "  font.set_text(s, 0.0, flags=flags)\n",
      "/usr/local/lib/python3.7/dist-packages/matplotlib/backends/backend_agg.py:214: RuntimeWarning: Glyph 39564 missing from current font.\n",
      "  font.set_text(s, 0.0, flags=flags)\n",
      "/usr/local/lib/python3.7/dist-packages/matplotlib/backends/backend_agg.py:214: RuntimeWarning: Glyph 35777 missing from current font.\n",
      "  font.set_text(s, 0.0, flags=flags)\n",
      "/usr/local/lib/python3.7/dist-packages/matplotlib/backends/backend_agg.py:183: RuntimeWarning: Glyph 35757 missing from current font.\n",
      "  font.set_text(s, 0, flags=flags)\n",
      "/usr/local/lib/python3.7/dist-packages/matplotlib/backends/backend_agg.py:183: RuntimeWarning: Glyph 32451 missing from current font.\n",
      "  font.set_text(s, 0, flags=flags)\n",
      "/usr/local/lib/python3.7/dist-packages/matplotlib/backends/backend_agg.py:183: RuntimeWarning: Glyph 38598 missing from current font.\n",
      "  font.set_text(s, 0, flags=flags)\n",
      "/usr/local/lib/python3.7/dist-packages/matplotlib/backends/backend_agg.py:183: RuntimeWarning: Glyph 39564 missing from current font.\n",
      "  font.set_text(s, 0, flags=flags)\n",
      "/usr/local/lib/python3.7/dist-packages/matplotlib/backends/backend_agg.py:183: RuntimeWarning: Glyph 35777 missing from current font.\n",
      "  font.set_text(s, 0, flags=flags)\n",
      "/usr/local/lib/python3.7/dist-packages/matplotlib/backends/backend_agg.py:214: RuntimeWarning: Glyph 27169 missing from current font.\n",
      "  font.set_text(s, 0.0, flags=flags)\n",
      "/usr/local/lib/python3.7/dist-packages/matplotlib/backends/backend_agg.py:214: RuntimeWarning: Glyph 22411 missing from current font.\n",
      "  font.set_text(s, 0.0, flags=flags)\n",
      "/usr/local/lib/python3.7/dist-packages/matplotlib/backends/backend_agg.py:214: RuntimeWarning: Glyph 25439 missing from current font.\n",
      "  font.set_text(s, 0.0, flags=flags)\n",
      "/usr/local/lib/python3.7/dist-packages/matplotlib/backends/backend_agg.py:214: RuntimeWarning: Glyph 22833 missing from current font.\n",
      "  font.set_text(s, 0.0, flags=flags)\n",
      "/usr/local/lib/python3.7/dist-packages/matplotlib/backends/backend_agg.py:214: RuntimeWarning: Glyph 20989 missing from current font.\n",
      "  font.set_text(s, 0.0, flags=flags)\n",
      "/usr/local/lib/python3.7/dist-packages/matplotlib/backends/backend_agg.py:214: RuntimeWarning: Glyph 25968 missing from current font.\n",
      "  font.set_text(s, 0.0, flags=flags)\n",
      "/usr/local/lib/python3.7/dist-packages/matplotlib/backends/backend_agg.py:214: RuntimeWarning: Glyph 21464 missing from current font.\n",
      "  font.set_text(s, 0.0, flags=flags)\n",
      "/usr/local/lib/python3.7/dist-packages/matplotlib/backends/backend_agg.py:214: RuntimeWarning: Glyph 21270 missing from current font.\n",
      "  font.set_text(s, 0.0, flags=flags)\n",
      "/usr/local/lib/python3.7/dist-packages/matplotlib/backends/backend_agg.py:214: RuntimeWarning: Glyph 26354 missing from current font.\n",
      "  font.set_text(s, 0.0, flags=flags)\n",
      "/usr/local/lib/python3.7/dist-packages/matplotlib/backends/backend_agg.py:214: RuntimeWarning: Glyph 32447 missing from current font.\n",
      "  font.set_text(s, 0.0, flags=flags)\n",
      "/usr/local/lib/python3.7/dist-packages/matplotlib/backends/backend_agg.py:183: RuntimeWarning: Glyph 27169 missing from current font.\n",
      "  font.set_text(s, 0, flags=flags)\n",
      "/usr/local/lib/python3.7/dist-packages/matplotlib/backends/backend_agg.py:183: RuntimeWarning: Glyph 22411 missing from current font.\n",
      "  font.set_text(s, 0, flags=flags)\n",
      "/usr/local/lib/python3.7/dist-packages/matplotlib/backends/backend_agg.py:183: RuntimeWarning: Glyph 25439 missing from current font.\n",
      "  font.set_text(s, 0, flags=flags)\n",
      "/usr/local/lib/python3.7/dist-packages/matplotlib/backends/backend_agg.py:183: RuntimeWarning: Glyph 22833 missing from current font.\n",
      "  font.set_text(s, 0, flags=flags)\n",
      "/usr/local/lib/python3.7/dist-packages/matplotlib/backends/backend_agg.py:183: RuntimeWarning: Glyph 20989 missing from current font.\n",
      "  font.set_text(s, 0, flags=flags)\n",
      "/usr/local/lib/python3.7/dist-packages/matplotlib/backends/backend_agg.py:183: RuntimeWarning: Glyph 25968 missing from current font.\n",
      "  font.set_text(s, 0, flags=flags)\n",
      "/usr/local/lib/python3.7/dist-packages/matplotlib/backends/backend_agg.py:183: RuntimeWarning: Glyph 21464 missing from current font.\n",
      "  font.set_text(s, 0, flags=flags)\n",
      "/usr/local/lib/python3.7/dist-packages/matplotlib/backends/backend_agg.py:183: RuntimeWarning: Glyph 21270 missing from current font.\n",
      "  font.set_text(s, 0, flags=flags)\n",
      "/usr/local/lib/python3.7/dist-packages/matplotlib/backends/backend_agg.py:183: RuntimeWarning: Glyph 26354 missing from current font.\n",
      "  font.set_text(s, 0, flags=flags)\n",
      "/usr/local/lib/python3.7/dist-packages/matplotlib/backends/backend_agg.py:183: RuntimeWarning: Glyph 32447 missing from current font.\n",
      "  font.set_text(s, 0, flags=flags)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZYAAAEoCAYAAAB7ONeTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3df5xVVb3/8deHGX7KCIgwIKCQoqYgoyDgVQy1AM0u2TXTMqU0vletm2mpZKbXtOymqN3UMiG1vKJZJBmGpExqhSiI+AMNwl+DCPLTGXHAGT7fP9Y+zJmZMz85c/aeOe/n47EfZ5+1197nc/gxn1l7rb2WuTsiIiLZ0inuAEREpGNRYhERkaxSYhERkaxSYhERkaxSYhERkaxSYhERkaxSYhERkaxSYpHYmNk1ZraxjT/jbjN7ri0/Q0RqK4w7AJE29gOge7YvamaDgMcbqTIDWKI6e1bH3ec2clwSSolFOjR3/1cbXboz8Ia7T6l7wMy+DfRSnazUkXZIt8IkscxshJn9yczKo+23ZjYgOtbZzJ43s1Izs7Rz/tfMNqbVq3crzMxKzOxxM9tuZlvM7D4zK047PtTM3MzOMLNfmNk2Myszs/82M/2fEWmC/pNIIpnZQcDfgG7A2cA04HDgj2Zm7v4RcA5wDPDN6JwTgYuAi9z93Qau2w8oBXoAXwS+AXwCWGhmXepU/x+gAjgd+A3w/WhfRBqhW2GSVFcD7wInu/tOADNbAbwKnAL8yd1fNLNrgB+a2dPAbOC37v5AI9e9NHqd7O7vR9ddBSwG/gO4P63uk+6eqr/QzKYAnwMezMYXFOmo1GKRpPokMBfYZWaFZlYIvA68AYxJq/c/wAvA04TWzYVNXHcs8FgqqQC4+zPRdY+rU/exOu9fAQa36FuI5CElFkmqfYHLgY/qbB8DhqQquXs18FugK/CAu29q4roDgfUZytcD+9Qp21rn/U5C8hKRRuhWmCTVZkKL5a4Mx3Y/+2JmBxBumz0P/KeZ/dLdX2rkuuuA/hnKi4GlrQ9XRFKUWCSpHid01i/1Blaji0aDzQbWEDrx/wLca2bjos79TJ4BLjCzIncvj65zNDCUcDtNRPaQEovErYuZZRppdSuwAPiTmc0mtFIGAZ8C7nb3UsIIsOOAo919h5lNI/S3XAlc08DnzQQuABaY2Y+BnsANwIvA77L0nUTymhKLxK2I0EdS1wnAeOA64E7C0/NrCS2Z1dFw5B8D/+3uKyA8DGlmlwO3mNk8d19W96Lu/p6ZnQDcRBgBthOYD3wrNfpMRPaMEovExt2voeGWRUpjz43sleGatwG3pb2flqHO88CJjcT1BmAZyutdS0Tq06gwERHJKrVYRFrnI2Comb3awPEZqpOVOtIOWQMDbkRERFpFt8JERCSrlFhERCSrlFhERCSrlFhERCSrlFhERCSrlFhERCSrlFhERCSrlFhERCSrlFhERCSrlFhERCSrlFhERCSrNAklsO+++/rQoUNbde4HH3zAXnvVm709UZIeY9Ljg+THmPT4IPkxJj0+SF6MS5cu3eju/eodcPe830aPHu2ttWjRolafmytJjzHp8bknP8akx+ee/BiTHp978mIEnvMMP1N1K0xERLJKiUVERLJKiUVERLJKnfciknc++ugjysrKqKys3F3Wq1cvVq5cGWNUTYsrxm7dujF48GA6d+7crPpKLCKSd8rKyigqKmLo0KGYGQDl5eUUFRXFHFnj4ojR3dm0aRNlZWUMGzasWefoVpiI5J3Kykr69u27O6lIw8yMvn371mrdNUWJRUTykpJK87X0z0qJZU/cdhv9n3gi7ihERBJFiWVPzJ7NgPnz445CRCRRlFj2REkJPVevBve4IxERYejQoWzcuDHuMDQqbI+UlNBl9mxYtw722y/uaESknbjmmmtYvHgxhYXhR3BVVRXjx48HaLQcoLCwcHf5Nddck/vgm0GJZU+UlITX5cuVWETaq4svhuXL6V5dDQUF2blmSQncckujVebMmUPv3r0B2Lp1K7dE9RsrLygooKioqFZ5Q2bOnMns2bMBOP/887n44ov54IMPOOOMMygrK6O6upqrrrqKL3zhC1xxxRXMmzePwsJCJk2axI033rhHX1+JZU8ccUR4Xb4cTjkl3lhERCJLly7lV7/6Fc888wzuzrhx4/jEJz7BmjVr2G+//fjTn/4EwLZt29i0aRNz587l1VdfxczYunXrHn9+ThOLmc0GTgU2uPuItPJvABcB1cCf3P2yqHwGcF5U/l/uviAqnwLcChQAd7n7DVH5MGAO0BdYCnzZ3Xe22Rfq1YsP99uP7suXt9lHiEgbi37z/7AdPCDZXE8//TSnnXba7in2P/e5z/HUU08xZcoULr30Ui6//HJOPfVUJkyYQFVVFd26deO8887j1FNP5dRTT93jz8915/3dwJT0AjM7AZgKjHL3w4Ebo/LDgDOBw6NzbjezAjMrAG4DTgYOA86K6gL8GLjZ3Q8CthCSUpuqOPDA0GIREUm4gw8+mGXLljFy5Ei+973vce2111JYWMiSJUs4/fTTeeSRR5gyZUrTF2pCThOLuz8JbK5TfAFwg7vviOpsiMqnAnPcfYe7vw6sBsZG22p3XxO1RuYAUy08wXMi8FB0/j3AZ9v0CxElltWroaKirT9KRKRZJkyYwB/+8Ae2b9/OBx98wNy5c5kwYQLvvPMOPXr04Oyzz+Y73/kOy5Yto6Kigm3btnHKKadw880388ILL+zx5yehj+VgYIKZXQ9UAt9292eBQcDitHplURnA23XKxxFuf21196oM9esxs+nAdIDi4mJKS0tbFfxegwczzJ1l99zD+4cf3qprtLWKiopWf79cSHp8kPwYkx4fJCvGXr16UV5eXqusurq6Xllb2bFjB+Xl5RREgwXKy8vZsWPH7v2GyouKiigvL99dXjded6eiooLhw4dz1llnMWbMGADOOeccDjroIP7yl79w1VVX0alTJwoLC7n55ptZt24dZ555Jjt27MDduf766zP+OVRWVjb77y8JiaUQ2AcYDxwNPGhmH2vrD3X3O4E7AcaMGeMTJ05s1XX+sX49AEd16gStvEZbKy0tpbXfLxeSHh8kP8akxwfJinHlypX1+lNyOcFj165dKSoq2v151dXVdO3aFaDR8tSosFR53XjffPPN3fszZsxgxowZtY6fdtppnHbaafXiWbp0aZMxd+vWjSOPPLJZ3y8JiaUM+H20zOUSM9sF7AusBYak1RscldFA+Sagt5kVRq2W9PptZkf//tCnj/pZRKTZ+vfvzznnnEOnTqE3YteuXbv7Nhor37VrF4WFhbXKkygJieUPwAnAIjM7GOgCbATmAf9nZjOB/YDhwBLAgOHRCLC1hA7+L7q7m9ki4HRCv8u5wMNtHr1ZGLOuxCIizXThhRdy4YUXNnisofL2MLU/5Ljz3szuB/4BHGJmZWZ2HjAb+JiZvUSUEDx4GXgQeAX4M3CRu1dHrZGvAwuAlcCDUV2Ay4FLzGw1oc9lVk6+2KhRsGIFVFU1XVdEpIPLaYvF3c9q4NDZDdS/Hrg+Q/l8oN7sj+6+hjBqLLdKSqCyElatgo9/POcfLyKSJJqEMhtSU7tkYZieiEh7p8SSDR//OHTurH4WERGUWLKjSxc4/HAlFhGJlabN72hKSuDRR+OOQkTaAU2bL81TUgJ33w3vvgsDBsQdjYg0UzRrPtXV3XM5a76mzZdmGDUqvC5fDgl+cElEOr68mja/Q0sllhdeUGIRaUdSv/iXl3/YLh4+bI58mza/4+rTBw44QB34IpJYHXLa/A5PU7uISAJo2vyOpKQE5s2DDz6AqAkqIpJrRx11FNOmTWPs2DARyfnnn8+RRx7JggUL+M53vkOnTp3o3Lkzd9xxB+Xl5UydOpXKykrcnZkzZ+7x5yuxZFNJCbjDSy/BuHFxRyMieeaNN97YvX/JJZdwySWX1Do+efJkJk+eXO+8JUuWZDUOJZZsSh8ZpsQiIg3QtPnSfEOHwt57a84wkXbA3Qkrmudee5s2PyyX1XzqvM8mrc0i0i5069aNTZs2tfgHZj5ydzZt2kS3bt2afY5aLNlWUgKzZkF1NVl7jFdEsmrw4MGUlZXx3nvv7S6rrKxs0Q/POMQVY7du3Rg8eHCz6+c0sZjZbOBUYIO7j6hz7FLgRqCfu2+00Ea9FTgF2A5Mc/dlUd1zge9Fp17n7vdE5aOBu4HuhPVavum5/pWkpCSMCvvXv+Dgg3P60SLSPJ07d2bYsGG1ykpLS5u9pntc2kOMkPtbYXcD9XqczGwIMAl4K634ZMJyxMOB6cAdUd19gKuBcYRFva42sz7ROXcAX0s7L/e9W6m1WXQ7TETyVE4Ti7s/CWzOcOhm4DIgvXUxFbg3WqZ4MdDbzAYCk4GF7r7Z3bcAC4Ep0bG93X1x1Eq5F/hsW36fjA47DAoLlVhEJG/F3nlvZlOBte5edyjVIODttPdlUVlj5WUZynOra9ew8JdGholInoq1897MegDfJdwGy/VnTyfcYqO4uJjS0tJWXaeioqLeuYcOGECfJUv4RyuvmW2ZYkySpMcHyY8x6fFB8mNMenzQPmIEwlCyXG7AUOClaH8ksAF4I9qqCP0sA4BfAGelnfcaMBA4C/hFWvkvorKBwKtp5bXqNbaNHj3aW2vRokX1C2+6yR3c169v9XWzKWOMCZL0+NyTH2PS43NPfoxJj889eTECz3mGn6mx3gpz9xfdvb+7D3X3oYTbV0e5+7vAPOAcC8YD29x9HbAAmGRmfaJO+0nAgujY+2Y2PhpRdg7wcCxfLNWBr9thIpKHcppYzOx+4B/AIWZWZmbnNVJ9PrAGWA38ErgQwN03Az8Ano22a6Myojp3Ref8C4hnreD0qV1ERPJMTvtY3P2sJo4PTdt34KIG6s0GZmcofw4YUf+MHOvbF4YMUWIRkbwU+6iwDmvUKN0KE5G8pMTSVkpK4NVX4cMP445ERCSnlFjaSklJmC/s5ZfjjkREJKeUWNqKpnYRkTylxNJWhg2DoiIlFhHJO0osbaVTp9CBr8QiInlGiaUtjRoFK1bArl1xRyIikjNKLG2ppATKy+H11+OOREQkZ5RY2pI68EUkDymxtKXDDw/LEyuxiEgeUWJpS927w6GHKrGISF5RYmlrJSVKLCKSV5RY2tqoUVBWBps2xR2JiEhOKLG0Na3NIiJ5RomlrWltFhHJM0osba1/f9hvPyUWEckbuV5BcraZbTCzl9LKfmJmr5rZCjOba2a9047NMLPVZvaamU1OK58Sla02syvSyoeZ2TNR+QNm1iV3364R6sAXkTyS6xbL3cCUOmULgRHufgTwT2AGgJkdBpwJHB6dc7uZFZhZAXAbcDJwGHBWVBfgx8DN7n4QsAVobOnj3CkpgZUrYceOuCMREWlzOU0s7v4ksLlO2WPuXhW9XQwMjvanAnPcfYe7v05Yx35stK129zXuvhOYA0w1MwNOBB6Kzr8H+GybfqHmKimBqip45ZW4IxERaXM5XfO+Gb4KPBDtDyIkmpSyqAzg7Trl44C+wNa0JJVevx4zmw5MByguLqa0tLRVAVdUVDR5bvfKSsYBr86Zw7vbtrXqc/ZEc2KMU9Ljg+THmPT4IPkxJj0+aB8xQoISi5ldCVQB9+Xi89z9TuBOgDFjxvjEiRNbdZ3S0lKaPLe6Gi64gEMrKzm0lZ+zJ5oVY4ySHh8kP8akxwfJjzHp8UH7iBESkljMbBpwKnCSu3tUvBYYklZtcFRGA+WbgN5mVhi1WtLrx6ugAI44Qh34IpIXYh9ubGZTgMuAf3f37WmH5gFnmllXMxsGDAeWAM8Cw6MRYF0IHfzzooS0CDg9Ov9c4OFcfY8mpUaG7c6bIiIdU66HG98P/AM4xMzKzOw84GdAEbDQzJab2c8B3P1l4EHgFeDPwEXuXh21Rr4OLABWAg9GdQEuBy4xs9WEPpdZOfx6jSspgfffhzfeiDsSEZE2ldNbYe5+VobiBn/4u/v1wPUZyucD8zOUryGMGkue9Kldhg2LNxYRkTYU+62wvDFiBHTqpH4WEenwlFhypUcPOPhgJRYR6fCUWHJJU7uISB5QYsmlkhJ4803YsiXuSERE2owSSy5pbRYRyQNKLLmkxCIieUCJJZeKi2HAAPWziEiHpsSSa6NGKbGISIemxJJrJSXw8suwc2fckYiItAklllwrKYGPPgoLf4mIdEBKLLmW6sDX7TAR6aCUWHJt+HDo3l0jw0Skw1JiyTWtzSIiHZwSSxxSI8O0NouIdEBKLHEoKQnTurz9dtyRiIhkXa4X+pptZhvM7KW0sn3MbKGZrYpe+0TlZmY/NbPVZrbCzI5KO+fcqP4qMzs3rXy0mb0YnfNTM7Ncfr9mUwe+iHRguW6x3A1MqVN2BfC4uw8HHo/eA5xMWI54ODAduANCIgKuBsYRFvW6OpWMojpfSzuv7mclw8iRYKbEIiIdUk4Ti7s/CWyuUzwVuCfavwf4bFr5vR4sBnqb2UBgMrDQ3Te7+xZgITAlOra3uy92dwfuTbtWsvTsGUaHaWSYiHRASehjKXb3ddH+u0BxtD8ISO+EKIvKGisvy1CeTFqbRUQ6qJyued8Ud3czy8lQKTObTrjFRnFxMaWlpa26TkVFRavO3b+oiI+tWcNTjzxCdc+erfrs5mptjLmS9Pgg+TEmPT5IfoxJjw/aR4yQjMSy3swGuvu66HbWhqh8LTAkrd7gqGwtMLFOeWlUPjhD/Yzc/U7gToAxY8b4xIkTG6raqNLSUlp17vbtMGsWE3r1ggkTWvXZzdXqGHMk6fFB8mNMenyQ/BiTHh+0jxghGbfC5gGpkV3nAg+nlZ8TjQ4bD2yLbpktACaZWZ+o034SsCA69r6ZjY9Gg52Tdq3k0cgwEemgctpiMbP7Ca2Nfc2sjDC66wbgQTM7D3gTOCOqPh84BVgNbAe+AuDum83sB8CzUb1r3T01IOBCwsiz7sCj0ZZMAwdCv35KLCLS4eQ0sbj7WQ0cOilDXQcuauA6s4HZGcqfA0bsSYw5YxZaLRoZJiIdTBJuheWvkhJ46aUwjb6ISAfRosRiZoVm1rVO2SQzuzj9yXhpppIS2LEDXnst7khERLKmpS2WB4iegAcws/8C/gz8CFhsZqdmMbaOb9So8Kp+FhHpQFqaWMYTOtVTvgPc5O7dgbuAK7MVWF445BDo2lWJRUQ6lJYmlr6Ep+Mxs5HAfsDPo2O/BQ7LXmh5oLAwzBumxCIiHUhLE8t6YGi0PwV4093/Fb3vDuzKUlz5IzUyTGuziEgH0dLE8lvgx2b2E+BywkSPKUcCq7IVWN4oKYGNG+Gdd+KOREQkK1r6HMsVwPvA0YRO/B+mHRtN6NyXlkh/An9QcufMFBFprhYlFnevAq5t4NjnshJRvjniiPC6fDl8+tPxxiIikgUtfY6lv5kNS3tvZjbdzG4xs89kP7w8UFQEBx6oDnwR6TBa2sdyN/CttPfXArcTOvLnmtm07ISVZ7Q2i4h0IC1NLEcBTwCYWSfgP4HvuvuhwPXAxdkNL0+UlMC//gXl5XFHIiKyx1qaWHoBm6L90cA+wH3R+yeAg7IUV34pKQnDjV98Me5IRET2WEsTSxk1D0F+GnjV3VOLafUCKrMVWF7R2iwi0oG0dLjxbOB/zOyThMQyI+3YeGBltgLLK4MGwT77KLGISIfQ0uHGPzKztYTnWL5B7TVR9iHMFyYtlVqbRYlFRDqAFq/H4u73uvs33H1WtBhXqvw/3f2e1gZiZt8ys5fN7CUzu9/MupnZMDN7xsxWm9kDZtYlqts1er86Oj407TozovLXzGxya+PJuZKS0MdSVRV3JCIie6TFiSVak+ULZva/ZnZf9HqGmbV6NUozGwT8FzDG3UcABcCZwI+Bm939IGALcF50ynnAlqj85qgeZnZYdN7hhCHQt5tZQWvjyqmSEqishFWaFUdE2rcWPyAJPAfcT+hj+Vj0Ogd41sz67UEshUD3KEH1ANYBJwIPRcfvAT4b7U+N3hMdP8nMLCqf4+473P11YDUwdg9iyh114ItIB9HSVsZMwtT54919SarQzI4Gfhcd/3JLg3D3tWZ2I/AW8CHwGLAU2BpNIwNhRFpqMq1BwNvRuVVmti2KaxCwOO3S6efUYmbTgekAxcXFlJaWtjRsACoqKlp9bq14qqqY0LkzZfPmsWbgwD2+XrpsxdhWkh4fJD/GpMcHyY8x6fFB+4gRAHdv9gZsBr7YwLEvAZtbcr20c/sQnoPpB3QG/gCcDaxOqzMEeCnafwkYnHbsX8C+wM+As9PKZwGnN/X5o0eP9tZatGhRq8+t56ij3CdNyt71IlmNsQ0kPT735MeY9Pjckx9j0uNzT16MwHOe4WdqS/tYugINPR5eDnRp4fVSPgm87u7vuftHwO+BY4HeaX03g4HUMzNrCYmG6Hjqwc3d5RnOSb5Ro+C55/QEvoi0ay1NLIuBy81sr/TC6P3l1L4N1RJvAePNrEfUV3IS8AqwCDg9qnMu8HC0Py96T3T8iSh7zgPOjEaNDQOGA7tv2SXeeefB1q1wwQVa+EtE2q2W9rFcSvhh/7aZPUZYUbI/MBkwYGJrgnD3Z8zsIWAZUAU8D9wJ/AmYY2bXRWWzolNmAb82s9WE23NnRtd52cweJCSlKuAid69uTUyxOPZYuOYa+P734aST4CtfiTsiEZEWa+kDksvN7GBCgjkaOIIweuvnwEx339jaQNz9auDqOsVryDCqy90rgc83cJ3rCRNitk/f/S4sWgRf/zqMHw8f/3jcEYmItEiLnz1x9/cIK0lKWygogN/8Jgw/PuMMWLIEunePOyoRkWZrMrGY2bNAs2/4u3v7eG4kyfbbD+69F04+Gb71Lfj5z+OOSESk2ZrTYnmZFiQWyZIpU+A734Gf/CT0t3w+450/EZHEaTKxuPu0HMQhmVx/PTz5JJx/PowZA8OGNX2OiEjMWjxXmORQ584wZ06Y/fjMM2HnzrgjEhFpkhJL0g0dCrNmhU78K6+MOxoRkSYpsbQH//Ef4aHJG2+ERx+NOxoRkUYpsbQXM2fCEUfAOefA2vYzS42I5B8llvaiWzd44AHYvh3OPhuq28+EAiKSX5RY2pNDD4Xbb4fSUrjuurijERHJSImlvTn3XPjyl+Haa+Gvf407GhGRepRY2qPbb4eDDoIvfhHeey/uaEREalFiaY969gz9LZs2wbRpsGtX3BGJiOymxNJelZTATTfB/Plwyy1xRyMispsSS3t24YVw2mlwxRXw7LNxRyMiAiixtG9m4an8gQPhC1+AbdvijkhEJDmJxcx6m9lDZvaqma00s2PMbB8zW2hmq6LXPlFdM7OfmtlqM1thZkelXefcqP4qMzu34U/sIPr0gfvvh7fegunTtaSxiMQuMYkFuBX4s7sfCowCVhIWFHvc3YcDj1OzwNjJhPXshwPTgTsAzGwfwiqU4wgrT16dSkYd2r/9W3iu5cEH4Ze/jDsaEclziUgsZtYLOJ5oTXt33+nuW4GpwD1RtXuAz0b7U4F7PVgM9DazgcBkYKG7b3b3LcBCYEoOv0p8LrsMJk2Cb34TXnwx7mhEJI+ZJ+DWiZmVAHcCrxBaK0uBbwJr3b13VMeALe7e28weAW5w96ejY48DlwMTgW7ufl1UfhXwobvfmOEzpxNaOxQXF4+eM2dOq2KvqKigZ8+erTo32zpv3syYr32NqqIilt5xB7uiJY2TFGMmSY8Pkh9j0uOD5MeY9PggeTGecMIJS919TL0D7h77BowBqoBx0ftbgR8AW+vU2xK9PgIcl1b+eHSNbwPfSyu/Cvh2U58/evRob61Fixa1+tw28Ze/uJu5f/Wru4sSF2MdSY/PPfkxJj0+9+THmPT43JMXI/CcZ/iZmohbYUAZUObuz0TvHwKOAtZHt7iIXjdEx9cCQ9LOHxyVNVSeP046Cb77XZg9G/7v/+KORkTyUCISi7u/C7xtZodERScRbovNA1Iju84FHo725wHnRKPDxgPb3H0dsACYZGZ9ok77SVFZfrnmGjjuOPh//w9WrYo7GhHJM4lILJFvAPeZ2QqgBPghcAPwKTNbBXwyeg8wH1gDrAZ+CVwI4O6bCbfQno22a6Oy/FJYGForXbrAmWfS6cMP445IRPJIYdwBpLj7ckI/SV0nZajrwEUNXGc2MDu70bVDQ4bA3XfD1KmMmT4dfv97OProuKMSkTyQpBaLZNtnPgNPPEGnnTvhmGPCsy5VVXFHJSIdnBJLRzdxIs/ddRd8/vNw1VUwcSK8/nrcUYlIB6bEkgeqiorCtC+/+U14eHLUKLjnHk3/IiJtQokln3zpS7BiBRx5ZFjH5QtfgM35N7ZBRNqWEku+OeAAeOIJ+NGPYO5cGDkS/vKXuKMSkQ5EiSUfFRSENVyeeQaKiuBTn4JLL4XKyrgjE5EOQIklnx11FCxbFhYMmzkTxo6Fl16KOyoRaeeUWPJdjx5w223wyCOwfj2MGQO33gq7dsUdmYi0U0osEnz602HE2KRJcPHFMGUKvPNO3FGJSDukxCI1+veHhx+Gn/8cnn46dOz/7ndxRyUi7YwSi9RmFiavfP55+NjH4PTT4atfhfLyuCMTkXZCiUUyO+QQ+Pvf4corw8OUJSXwj3/EHZWItANKLNKwzp3D/GJ//WvozD/uOLj6avjoo7gjE5EEU2KRph13HCxfDmefDddeC/vvD5ddBitXxh2ZiCSQEos0T69e4ZbYo4+G511mzoTDDoNx40Jn/9atcUcoIgmRqMRiZgVm9ryZPRK9H2Zmz5jZajN7wMy6ROVdo/ero+ND064xIyp/zcwmx/NNOrApU8LIsbVr4aabYPt2uOACGDAAzjoLHnsMqqvjjlJEYpSoxAJ8E0i/v/Jj4GZ3PwjYApwXlZ8HbInKb47qYWaHAWcChwNTgNvNrCBHseeX4mK45JIwqeVzz8H558OCBTB5MgwdGjr9tSyySF5KTGIxs8HAp4G7ovcGnAg8FFW5B/hstD81ek90/KSo/lRgjrvvcPfXCUsXj83NN8hTZjB6NPzsZ+GBygcfDM+/3HADHHwwTJgAs2ZpuLJIHklMYgFuAS4DUnOJ9I8wF9kAABQOSURBVAW2untqycMyYFC0Pwh4GyA6vi2qv7s8wznS1rp1CwuKzZ8Pb78dkst774XWzIABcM45sGiRposR6eASsea9mZ0KbHD3pWY2MUefOR2YDlBcXExpaWmrrlNRUdHqc3MlthjHjYOxY9l75UoGPPoo/X//ewp//Ws+HDCA9ZMn8+7kyVQOHKg/wyxIenyQ/BiTHh+0jxgBcPfYN+BHhNbFG8C7wHbgPmAjUBjVOQZYEO0vAI6J9gujegbMAGakXXd3vca20aNHe2stWrSo1efmSmJi3L7d/b773D/5SXczd3A/4QR/9ZJL3F991X3XrrgjbFBi/gwbkPT43JMfY9Ljc09ejMBznuFnaiJuhbn7DHcf7O5DCZ3vT7j7l4BFwOlRtXOBh6P9edF7ouNPRF9yHnBmNGpsGDAcWJKjryFN6d4dvvhFWLgQ3ngDfvADeOstDpk5Ew49NNwuO/10+OlPw3MzGl0m0i4l4lZYIy4H5pjZdcDzwKyofBbwazNbDWwmJCPc/WUzexB4BagCLnJ3/XRKov33h+99D668kmd+8xvG7dgBTz4ZttTEl3vvHR7OPP74MAhgzBjo0iXeuEWkSYlLLO5eCpRG+2vIMKrL3SuBzzdw/vXA9W0XoWSVGR8OGQITJ4ZOfoC33oKnngpJ5qmnwmAACC2e8eNDkjn++LC/116xhS4imSUusYiw//7wpS+FDWDDhjCNfyrRXHddGFlWWBiGOh9/fNiOPRb69Ik3dhFRYpF2oH9/+NznwgawbVuYeTnVqrnlFvjJT8IzNSNH1rRoJkyAgQPjjV0kDymxSPvTqxecfHLYAD78EJYsqemj+dWvwnLLAAcdVJNkjj8ehg0LCUhE2owSi7R/3bvDJz4RNgjT+j//fM2ts7lzYfbscGy//WonmsMOg06JGBwp0mEosUjH07lzmIF57Fj49rdDf8wrr9S0aJ58EubMCXX32af2yLMjjwzni0irKbFIx9epE4wYEbYLLwR3WLOm9sizefNC3b32gmOOqRkQMHZsaBGJSLMpsUj+MYMDDwzbtGmhbN262onm6qtDAurcGY4+mkOLisKSAPvtV3sbMEDP1ojUocQiAmH02BlnhA1gyxb4299Covn73+n9/PPw+ONQVVX/3H79YNCg+kknfevfHwq0goPkByUWkUz69IFTTw0bsLi0lInHHw8bN4blAdauDa91t2XLYP360NpJ16lTaN2kEs3QoTB8eFha4OCDYcgQJR7pMJRYRJqrU6fQ8ujfH0pKGq5XVRWSS6bE88478Prr8MQTUFFRc07XruHWXCrRHHxwTeIpLtYQaWlXlFhEsq2wMNwaG9TIUkDu8O67YZXNf/6zZnvttTCFzc6dNXWLiuonm9R+795t/31EWkiJRSQOZqFfZ+DAMPosXXV1mC8tlWxSyWfx4jBMOv02W79+cPDBHNqjB/zxj7DvvqFs331rtn79wq09Pa8jOaLEIpI0BQVhhoBhw2Dy5NrHKivDUOk6LZ3eK1aEwQbbt2e+ZqdO4ZmduoknUxJKvWqCT2klJRaR9qRbtzBbwGGH1SpeXFrKxIkTQ2LZtCksCb1xY81W9/0//xnmW9u4seF1b/r2hY99LCS41Gtqf//99SCpNEiJRaQj6dEjbEOGNK++e5jUs27iWb8+LMb2+uthpNvcuWGqnJROncJnpCed9Nf+/TXgII8psYjkM7MwAKB37zAYoCHV1WGI9euvh1tx6a/z54eBCOl69Kjdwhk2jOL33oPNm8MtttTWo0ft9507KyF1AIlILGY2BLgXKAYcuNPdbzWzfYAHgKHAG8AZ7r7FzAy4FTgF2A5Mc/dl0bXOBb4XXfo6d78nl99FpEMqKAi3v/bfv2ayz3Tbt9e0cOomntJSqKjg4839nMYST/r7Xr1Cy6i4uPZr795KTjFLRGIhLCN8qbsvM7MiYKmZLQSmAY+7+w1mdgVwBWG54pMJ69kPB8YBdwDjokR0NTCGkKCWmtk8d9+S828kkk969MjY9wOE222bNvHMn//MuBEjQhL64IOwpe839X7Dhtrvy8vrP4gKodWTnmgyJZ/Ua79+6itqA4lILO6+DlgX7Zeb2UpgEDAVmBhVu4ewZPHlUfm97u7AYjPrbWYDo7oL3X0zQJScpgD35+zLiEhtZrDvvnw4eHDjD5a2VHV16A/asCH0CTX0+sor4XXHjszX2WcfKC6mpHPn0CIrKqq97b13/bK65V27qpWUxjxTxo+RmQ0FngRGAG+5e++o3IAt7t7bzB4BbnD3p6NjjxMSzkSgm7tfF5VfBXzo7jdm+JzpwHSA4uLi0XNS06i3UEVFBT179mzVubmS9BiTHh8kP8akxwcxx+hOwfbtdNmyhc5bttBlyxa6bN1as79lC502baLLzp0UbN9O4fbtFGzfTkFDyaiOXQUFVPfoQXWPHlRFr9Xdu1O1115U9exZs6W9r04v79mT6m7dmkxOSft7PuGEE5a6+5i65YlosaSYWU/gd8DF7v6+pf0hu7ubWdayoLvfCdwJMGbMGJ84cWKrrlOaGuaZYEmPMenxQfJjTHp8kPwYM8ZXVRWm3ikvD9v779fsp5V1Ki+nU3k5nevWe/dd2Lo1jLyrrGw8gIKC0G/Uu3fNa2qL3q9ev56DRo4MSzn06BFeU1v6+/T9GOagS0xiMbPOhKRyn7v/Pipeb2YD3X1ddKtrQ1S+FkgfTzk4KltLza2zVHlpW8YtIh1YYWHND/c9VVkZEsy2bSHZpBJOY/urVtXsl5dzUGs+t0uXxpPP3LlZX3MoEYklus01C1jp7jPTDs0DzgVuiF4fTiv/upnNIXTeb4uSzwLgh2bWJ6o3CZiRi+8gItKobt3CVlzcuvOrqnhqwQImjBkTBjF8+GHNlv6+Jftbt7bJ4IVEJBbgWODLwItmtjwq+y4hoTxoZucBbwLRYhnMJww1Xk0YbvwVAHffbGY/AJ6N6l2b6sgXEWnXCgup3muv1iemHEpEYok64RvqtTopQ30HLmrgWrOB2dmLTkREWkLTnYqISFYpsYiISFYpseyBDRtg1664oxARSZZE9LG0V5MmwerVxzF2LIwdC0cfHV4HD9ZDuCKSv5RY9sC3vw0PPbSetWsHMXNmzazixcW1E83RR4dZI0RE8oESyx44+2wYPHgVEycOYscOeOEFePZZWLIkbI88UjNH3oEH1iSasWPhyCPD80kiIh2NEkuWdO1akzQuigZCb9sGS5fWJJu//S0sWQ5hloURI2q3akaMCA/6ioi0Z/ox1oZ69YITTwxbyrvv1iSaZ5+F3/0O7rorHOveHT7+8TCbd9++NcuQZ9rv2zckMxGRpFFiybEBA+AznwkbhFtla9bUJJqVK8OS5a+9FmYELy9v+Fo9ezaefFL7q1b1ZNCgkOj23lszfItI21JiiZlZ6H858EA466z6x3fuDIlm48b6r3X3V60Kr++/X/cqtWe17tw5JJhUomnufup9t27hGnW3Thq8LiIosSRely4wcGDYmmvnzrC0eCrxPP30ixxwwEjefz/0+7z/PvX23367dnlqhFtLFBTUTzZdumROQunl27aNpLg4nJ++FRZmt6yx8qaOvfZaEXWXwcjU6mtuWSaZlkZqbtlrrxVRVBSSu1nzXpuq4x6e03KvvV/3tbnHVq/uSb9+4c859Xef2q9blo1fUtzDWmAffRS2qqqa/UzbK6/sTfqSKGbN32/oWHP/naWXtfS7p75nVVXY0vebsx1zTPZ/KVRi6YC6dAm33AYMCO/dN9GSZTDcw2J7DSWiysr6/yl37sz8n7Wp8u3bYcuWLuzYUfOfIn2rW5apTnV1m/wx1jE6Fx+yB5IeH9RtOTemU6eGk07q1azppNEyR7X0hDbTUPLZufNYzGonhj39919Zmf3+WiUWqcesZobv/v3b/vNKS5fu0QJQqd+Km0o+zTneUHJ78cUXGTlyZK3PzBRHc8saasW0thW0YsWLjBgxstHWQ6bXxo6lWi6NtWqae8wMVqx4iUMPHVEvGaQnhbpljR1zr90KTk9Ambamjr/88gqOOOKI3X9Hqb+7pvYbO7ZrV+t/YcpUZ926DRxwwKDdiaewMDtbtimxSLuXfsuhrfTs2bJWX64lPT6AXr02JjrGnj03Jzo+gNLS8Nxc0qm7VUREskqJRUREsqpDJhYzm2Jmr5nZajO7Iu54RETySYdLLGZWANwGnAwcBpxlZofFG5WISP7ocIkFGAusdvc17r4TmANMjTkmEZG8YZ5pPGQ7ZmanA1Pc/fzo/ZeBce7+9Tr1pgPTAYqLi0fPSc0O2UIVFRX0rPvkXMIkPcakxwfJjzHp8UHyY0x6fJC8GE844YSl7l7vAaW8HW7s7ncCdwKMGTPGW/scRWlp6R49g5ELSY8x6fFB8mNMenyQ/BiTHh+0jxihY94KWwsMSXs/OCoTEZEc6Ii3wgqBfwInERLKs8AX3f3lRs55D3izlR+5L7CxlefmStJjTHp8kPwYkx4fJD/GpMcHyYvxAHfvV7eww90Kc/cqM/s6sAAoAGY3llSic+r9wTSXmT2X6R5jkiQ9xqTHB8mPMenxQfJjTHp80D5ihA6YWADcfT4wP+44RETyUUfsYxERkRgpsey5O+MOoBmSHmPS44Pkx5j0+CD5MSY9PmgfMXa8znsREYmXWiwiIpJVSiytlPSJLs1siJktMrNXzOxlM/tm3DFlYmYFZva8mT0SdyyZmFlvM3vIzF41s5VmdkzcMdVlZt+K/o5fMrP7zaxbAmKabWYbzOyltLJ9zGyhma2KXvskLL6fRH/PK8xsrpn1jiu+hmJMO3apmbmZ7RtHbE1RYmmFdjLRZRVwqbsfBowHLkpgjADfBFbGHUQjbgX+7O6HAqNIWKxmNgj4L2CMu48gDLE/M96oALgbmFKn7ArgcXcfDjwevY/L3dSPbyEwwt2PIDwLNyPXQdVxN/VjxMyGAJOAt3IdUHMpsbRO4ie6dPd17r4s2i8n/EBM1NJzZjYY+DRwV9yxZGJmvYDjgVkA7r7T3bfGG1VGhUD36OHgHsA7MceDuz8JbK5TPBW4J9q/B/hsToNKkyk+d3/M3auit4sJs3bEpoE/Q4CbgcuAxHaQK7G0ziDg7bT3ZSTsh3Y6MxsKHAk8E28k9dxC+A+yK+5AGjAMeA/4VXS77i4z2yvuoNK5+1rgRsJvr+uAbe7+WLxRNajY3ddF++8CxXEG04SvAo/GHURdZjYVWOvuL8QdS2OUWDo4M+sJ/A642N3fjzueFDM7Fdjg7kvjjqURhcBRwB3ufiTwAfHevqkn6qeYSkiC+wF7mdnZ8UbVNA/DURP5G7eZXUm4lXxf3LGkM7MewHeB78cdS1OUWFqnXUx0aWadCUnlPnf/fdzx1HEs8O9m9gbhVuKJZvabeEOqpwwoc/dUS+8hQqJJkk8Cr7v7e+7+EfB74N9ijqkh681sIED0uiHmeOoxs2nAqcCXPHnPYhxI+AXihej/zWBgmZkNiDWqDJRYWudZYLiZDTOzLoTO0nkxx1SLmRmhb2Clu8+MO5663H2Guw9296GEP78n3D1Rv2m7+7vA22Z2SFR0EvBKjCFl8hYw3sx6RH/nJ5GwAQZp5gHnRvvnAg/HGEs9ZjaFcGv23919e9zx1OXuL7p7f3cfGv2/KQOOiv6dJooSSytEHXypiS5XAg82NdFlDI4FvkxoCSyPtlPiDqod+gZwn5mtAEqAH8YcTy1Ra+ohYBnwIuH/dOxPZ5vZ/cA/gEPMrMzMzgNuAD5lZqsILa0bEhbfz4AiYGH0/+XnccXXSIztgp68FxGRrFKLRUREskqJRUREskqJRUREskqJRUREskqJRUREskqJRaQDMrOJ0ey3I+KORfKPEouIiGSVEouIiGSVEotIFpnZBDP7q5ltN7NNZvZLMyuKjk2Lbk8dbWZPmdmHZvZPMzstw3W+Hi2ItSNaTO5bGeocYWZ/NLOtZlZhZkvM7FN1qu1rZr+Njq8xswvb6KuL7KbEIpIlZnYs8BfClPCnAxcDpwC/qlP1AcI8WZ8jTMPyWzMblXadrwH/S5hb6zPAb4GbLG2lUjM7FPgbMBD4T+A0YC61J0cF+CXwQnS8FLjNzMbu+bcVaZimdBHJEjN7Cqhy9xPSyk4krJY4EhhDSDJXuvsPo+OdCBNbLnf3M6P3bwOPuftX0q5zO/AlwpomldE8UhOA4e7+YYZYJgKLgB+4+/ejss6ERcBmuXuipv+XjkUtFpEsiNbKOAZ40MwKUxvwNPARMDqt+tzUjrvvIrReUq2IwYR1VX5b5yMeAPYmJCiAE4EHMiWVOnYv+hVNq7+KmFdGlI5PiUUkO/oQ1pu/nZBIUtsOoDO1b1HVXYdkA+GWFmmv6+vUSb3fJ3rtS1gxsil1l1LeCXRrxnkirVYYdwAiHcRWwoqI1wDzMxx/B5gU7fcHNqUd609NkliXVpYutYxvag30TdQkIZFEUYtFJAvc/QNgMXCIuz+XYXsnrfruUWBRn8pUYElUVEZIQp+v8xFnAO8TOvsh9NucYWZqfUjiqMUikj2XAY+b2S7C4lvlwP7Ap4Er0+qdb2Y7gZeA84GDgLMg9LmY2TXAL8xsE7AQ+ARwAfBdd6+MrvHfhJVMnzSzmwgtmCOBTe4+u02/pUgT1GIRyRJ3fxo4HugH/Br4IyHZvE3tPpMzCa2WPwCjgC+4+/Np1/kl8M2oziOEpHOpu9+QVuc14DhgI3AXYUDA6cCbbfT1RJpNw41FcsTMphGGGxe5e0XM4Yi0GbVYREQkq5RYREQkq3QrTEREskotFhERySolFhERySolFhERySolFhERySolFhERySolFhERyar/DzcMYltSIinrAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "fig, ax = plt.subplots()#创建一个figure \n",
    "train_lossss = train_log\n",
    "dev_lossss = dev_log\n",
    "ax.plot(train_lossss, '-r',label='训练集 loss')\n",
    "ax.plot(dev_lossss, '-b',label='验证集 loss')\n",
    "\n",
    "####打开网格\n",
    "ax.grid(True)\n",
    "\n",
    "####定义x, y轴的名称\n",
    "ax.set_xlabel('epoch',fontsize=15)\n",
    "ax.set_ylabel('loss',fontsize=15)\n",
    "\n",
    "####定义标题\n",
    "fig.suptitle('Lexion模型损失函数变化曲线',fontsize=15)\n",
    "\n",
    "####展示图例 legend loc=是用来定义图例的位置的，还有很多选择，大家可以自己尝试\n",
    "ax.legend(loc = 'upper right')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "executionInfo": {
     "elapsed": 372,
     "status": "ok",
     "timestamp": 1651149632312,
     "user": {
      "displayName": "YING XU",
      "userId": "11756690436182978895"
     },
     "user_tz": -480
    },
    "id": "XWJ6e7NsX9QD"
   },
   "outputs": [],
   "source": [
    "gold_test = json.load(open('/content/drive/MyDrive/Colab Notebooks/LexionAN-master-master/save_model/LAB2/test_gold-lab2.json','r'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 2,
     "status": "ok",
     "timestamp": 1651149633869,
     "user": {
      "displayName": "YING XU",
      "userId": "11756690436182978895"
     },
     "user_tz": -480
    },
    "id": "_I0UDkgLZjQn",
    "outputId": "b2567190-4645-4aa8-b765-c3530dc32254"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['B-DRU',\n",
       " 'E-DRU',\n",
       " 'B-DRU',\n",
       " 'E-DRU',\n",
       " 'B-DRU',\n",
       " 'E-DRU',\n",
       " 'B-DRU',\n",
       " 'E-DRU',\n",
       " 'O',\n",
       " 'O']"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gold_test[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 2,
     "status": "ok",
     "timestamp": 1651149635312,
     "user": {
      "displayName": "YING XU",
      "userId": "11756690436182978895"
     },
     "user_tz": -480
    },
    "id": "6CIXykXBYEm9",
    "outputId": "19fd1339-20e3-42eb-f586-254fac5ff905"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3177"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(gold_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "executionInfo": {
     "elapsed": 387,
     "status": "ok",
     "timestamp": 1651149637923,
     "user": {
      "displayName": "YING XU",
      "userId": "11756690436182978895"
     },
     "user_tz": -480
    },
    "id": "Nt5SXZWPYmx5"
   },
   "outputs": [],
   "source": [
    "pred_test = json.load(open('/content/drive/MyDrive/Colab Notebooks/LexionAN-master-master/save_model/LAB2/test_log-lab2.json','r'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 2,
     "status": "ok",
     "timestamp": 1651149639278,
     "user": {
      "displayName": "YING XU",
      "userId": "11756690436182978895"
     },
     "user_tz": -480
    },
    "id": "mTr9wyu_bF4-",
    "outputId": "8c9a0511-1cc1-4a0d-f701-2e9cb6b10e09"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "120"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(pred_test[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "executionInfo": {
     "elapsed": 1,
     "status": "ok",
     "timestamp": 1651149641105,
     "user": {
      "displayName": "YING XU",
      "userId": "11756690436182978895"
     },
     "user_tz": -480
    },
    "id": "7vpAKcHPZV3b"
   },
   "outputs": [],
   "source": [
    "p_test = []\n",
    "for i in pred_test[-1]:\n",
    "  p_test.extend(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 2,
     "status": "ok",
     "timestamp": 1651149642572,
     "user": {
      "displayName": "YING XU",
      "userId": "11756690436182978895"
     },
     "user_tz": -480
    },
    "id": "ggTzkkpMYM7-",
    "outputId": "2d5a89f4-d403-41c8-c7f5-4da61fd171c2"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3177"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(p_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 2,
     "status": "ok",
     "timestamp": 1651149643975,
     "user": {
      "displayName": "YING XU",
      "userId": "11756690436182978895"
     },
     "user_tz": -480
    },
    "id": "NCmxdPsNYP1H",
    "outputId": "b0523f65-7826-4e59-9027-b73e3cbadc06"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "       B-DRU     0.8904    0.9186    0.9042       221\n",
      "      B-NAME     0.5667    0.8500    0.6800        20\n",
      "       B-QUA     0.9150    0.9396    0.9272       149\n",
      "       B-SYM     0.7818    0.7818    0.7818       110\n",
      "       B-USE     0.8821    0.8731    0.8776       197\n",
      "       E-DRU     0.8874    0.9206    0.9037       214\n",
      "      E-NAME     0.4828    0.7000    0.5714        20\n",
      "       E-QUA     0.9013    0.9514    0.9257       144\n",
      "       E-SYM     0.7963    0.7890    0.7926       109\n",
      "       E-USE     0.8425    0.8662    0.8542       142\n",
      "       I-DRU     0.9130    0.6000    0.7241        35\n",
      "      I-NAME     0.7679    0.9556    0.8515        45\n",
      "       I-QUA     0.7059    0.4800    0.5714        25\n",
      "       I-SYM     0.7699    0.8762    0.8196       210\n",
      "       I-USE     0.8568    0.8467    0.8517       424\n",
      "           O     0.9257    0.8741    0.8992      1112\n",
      "\n",
      "    accuracy                         0.8706      3177\n",
      "   macro avg     0.8053    0.8264    0.8085      3177\n",
      "weighted avg     0.8746    0.8706    0.8710      3177\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "print(classification_report(gold_test, p_test,digits=4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "zG1HWxFeYeRL"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "Softlexicon-Lab2.ipynb",
   "provenance": [
    {
     "file_id": "1NODPy_Ux1oJFL1o4xjQNH3rlVnhO6LU3",
     "timestamp": 1650113701068
    }
   ]
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
